diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index a249d127ca..98cd8e2777 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -366,6 +366,18 @@ stages: k8sVersion: "" dependsOn: ["test"] + # Cilium EBPF Dualstack Overlay E2E tests + - template: singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml + parameters: + name: "cilium_ebpf_ds_e2e" + displayName: Cilium EBPF on AKS DualStack Overlay + os: linux + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cilbpfdse2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: ["test"] + # Cilium Overlay with hubble E2E tests - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml parameters: @@ -503,6 +515,7 @@ stages: - cilium_nodesubnet_e2e - cilium_overlay_e2e - cilium_ebpf_overlay_e2e + - cilium_ebpf_ds_e2e - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e - aks_swift_vnetscale_e2e @@ -543,6 +556,10 @@ stages: name: cilium_ebpf_overlay_e2e clusterName: "cilbpfovere2e" region: $(REGION_AKS_CLUSTER_TEST) + cilium_ebpf_ds_e2e: + name: cilium_ebpf_ds_e2e + clusterName: "cilbpfdse2e" + region: $(REGION_AKS_CLUSTER_TEST) cilium_h_overlay_e2e: name: cilium_h_overlay_e2e clusterName: "cilwhleovere2e" diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml new file mode 100644 index 0000000000..95cb155373 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml @@ -0,0 +1,87 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + os: "linux" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + osSKU: "AzureLinux" + region: $(REGION_AKS_CLUSTER_TEST) + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + IPV6_IMAGE_REGISTRY: acnpublic.azurecr.io + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cilium EBPF Dualstack Test Suite - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: cilium-dualstack-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 50 + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + dependsOn: ${{ parameters.name }} + datapath: true + dns: true + portforward: true + service: true + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml new file mode 100644 index 0000000000..7a19cdc3c7 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml @@ -0,0 +1,142 @@ +parameters: + name: "" + clusterName: "" + scaleup: "" + + +steps: + - template: ../../templates/setup-environment.yaml + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + kubectl cluster-info + kubectl get pods -Aowide + + # see makefile + export AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY=acnpublic.azurecr.io + export AZURE_IPTABLES_MONITOR_TAG=$(make azure-iptables-monitor-version) + export AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY=acnpublic.azurecr.io + export AZURE_IP_MASQ_MERGER_TAG=$(make azure-ip-masq-merger-version) + make -C ./hack/aks deploy-ebpf-dualstack-cilium + + kubectl get pods -Aowide + name: "installCilium" + displayName: "Install Cilium EBPF on AKS Dualstack Overlay" + + - template: ../../templates/cilium-cli.yaml + + + - script: | + echo "Start Azilium E2E Tests on Overlay Cluster" + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true + retryCountOnTaskFailure: 3 + name: "aziliumTest" + displayName: "Run Azilium E2E on AKS Overlay" + + - script: | + kubectl get pods -A + echo "Waiting < 2 minutes for cilium to be ready" + # Ensure Cilium is ready Xm\Xs + cilium status --wait --wait-duration 2m + kubectl get crd -A + retryCountOnTaskFailure: 3 + name: "CiliumStatus" + displayName: "Cilium Status" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + + - template: ../../templates/cilium-connectivity-tests.yaml + + - script: | + ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` + echo "##vso[task.setvariable variable=ciliumNamespace]$ns" + retryCountOnTaskFailure: 3 + name: "nsCapture" + displayName: "Capture Connectivity Test Namespace" + + - script: | + set -e + kubectl get po -owide -A + cd test/integration/datapath + echo "Dualstack Overlay Linux datapath IPv6 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true + echo "Dualstack Overlay Linux datapath IPv4 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + retryCountOnTaskFailure: 3 + name: "DualStack_Overlay_Linux_Tests" + displayName: "DualStack Overlay Linux Tests" + + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl get pod -owide -A + cd test/integration/load + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns $(ciliumNamespace) + kubectl get pod -owide -A + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + name: "validatePods" + displayName: "Validate Pods" + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" + + - script: | + cd hack/scripts + chmod +x async-delete-test.sh + ./async-delete-test.sh + if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then + kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' + fi + name: "testAsyncDelete" + displayName: "Verify Async Delete when CNS is down" + + - template: ../../templates/cilium-mtu-check.yaml diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 3b34c16553..549d02d061 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -31,6 +31,7 @@ jobs: echo "Install az cli extension preview" az extension add --name aks-preview az extension update --name aks-preview + export POD_CIDRS="10.244.0.0/16,fdd5:a27a:b4bc:99d6::105/64" fi if ! [ -z ${K8S_VERSION} ]; then diff --git a/.pipelines/templates/log-template.yaml b/.pipelines/templates/log-template.yaml index 17a9d25ab7..9600e0d16d 100644 --- a/.pipelines/templates/log-template.yaml +++ b/.pipelines/templates/log-template.yaml @@ -186,7 +186,7 @@ steps: echo "Directory created: $(acnLogs)/"$node"_logs/Cilium-output/" file="cilium-endpoint.json" - kubectl exec -i -n kube-system $pod -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file + kubectl exec -i -n kube-system $pod -c cilium-agent -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file echo "Cilium, $file, captured: $(acnLogs)/"$node"_logs/Cilium-output/$file" done fi diff --git a/.pipelines/templates/log.steps.yaml b/.pipelines/templates/log.steps.yaml index 5451fe218b..d8bdfa3eb2 100644 --- a/.pipelines/templates/log.steps.yaml +++ b/.pipelines/templates/log.steps.yaml @@ -190,7 +190,7 @@ steps: echo "Directory created: $(acnLogs)/"$node"_logs/Cilium-output/" file="cilium-endpoint.json" - kubectl exec -i -n kube-system $pod -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file + kubectl exec -i -n kube-system $pod -c cilium-agent -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file echo "Cilium, $file, captured: $(acnLogs)/"$node"_logs/Cilium-output/$file" done fi diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5e1c8f3f9b..6dcccb5678 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -52,7 +52,7 @@ COMMON_AKS_FIELDS = $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --no-ssh-key \ --os-sku $(OS_SKU) \ $(LTS_ARGS) -POD_CIDR = 192.168.0.0/16 +POD_CIDR ?= 192.168.0.0/16 ##@ Help @@ -411,6 +411,7 @@ dualstack-byocni-nokubeproxy-up: rg-up ipv4 ipv6 overlay-net-up ## Brings up a D --network-plugin-mode overlay \ --subscription $(SUB) \ --ip-families ipv4,ipv6 \ + --pod-cidrs "$(POD_CIDRS)" \ --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview,AKSHTTPCustomFeatures=Microsoft.ContainerService/Ubuntu2404Preview \ --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes diff --git a/hack/aks/deploy.mk b/hack/aks/deploy.mk index 3b02fad9ee..d6ec1cbe03 100644 --- a/hack/aks/deploy.mk +++ b/hack/aks/deploy.mk @@ -9,23 +9,33 @@ AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY ?= mcr.microsoft.com/containernetworking AZURE_IP_MASQ_MERGER_TAG ?= v0.0.1-0 # so we can use in envsubst export IPV6_HP_BPF_VERSION +export IPV6_IMAGE_REGISTRY export AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY export AZURE_IPTABLES_MONITOR_TAG export AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY export AZURE_IP_MASQ_MERGER_TAG +export IPV6_HP_BPF_VERSION deploy-common-ebpf-cilium: @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-agent/files/ @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-operator/files/ # set cilium version tag and registry here so they are visible as env vars to envsubst CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ - envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION}' < \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${IPV}' < \ ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-operator/templates/deployment.yaml \ | kubectl apply -f - @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/common/ciliumclusterwidenetworkpolicies.yaml @kubectl wait --for=condition=Established crd/ciliumclusterwidenetworkpolicies.cilium.io @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/common/ +deploy-ebpf-dualstack-cilium: deploy-common-ebpf-cilium + @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/static/ + CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${IPV6_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_TAG},$${AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY},$${AZURE_IP_MASQ_MERGER_TAG}' < \ + ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/cilium.yaml \ + | kubectl apply -f - + @$(MAKE) wait-for-cilium + deploy-ebpf-overlay-cilium: deploy-common-ebpf-cilium @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/overlay/static/ CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml new file mode 100644 index 0000000000..cfec665ba8 --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml @@ -0,0 +1,475 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: cilium + name: cilium + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + labels: + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /etc/config + name: azure-ip-masq-dir + readOnly: true + - command: + - ./azure-ip-masq-merger + - -v + - "2" + image: $AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY/azure-ip-masq-merger:$AZURE_IP_MASQ_MERGER_TAG + imagePullPolicy: IfNotPresent + name: azure-ip-masq-merger + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config/ + name: azure-ip-masq-agent-config-volume + - mountPath: /etc/merged-config/ + name: azure-ip-masq-dir + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - /azure-block-iptables + - -mode=attach + - -overwrite=true + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: iptables-blocker-init + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /proc + name: hostproc + - command: + - /install-plugin.sh + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: install-cni-binaries + resources: {} + securityContext: + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-cgroup + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + imagePullPolicy: IfNotPresent + name: clean-cilium-state + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/etc/systemd + name: host-etc-systemd + - mountPath: /host/lib/systemd + name: host-lib-systemd + readOnly: true + - mountPath: /host/usr/lib + name: host-usr-lib + readOnly: true + - command: + - /ipv6-hp-bpf + image: $IPV6_IMAGE_REGISTRY/ipv6-hp-bpf:$IPV6_HP_BPF_VERSION + imagePullPolicy: IfNotPresent + name: start-ipv6-hp-bpf + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: ipv6-hp-bpf + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/log + type: DirectoryOrCreate + name: ipv6-hp-bpf + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - configMap: + defaultMode: 420 + name: allowed-iptables-patterns + optional: true + name: iptables-config + - hostPath: + path: /sys/fs/bpf/azure-block-iptables + type: DirectoryOrCreate + name: iptables-block-bpf-map + - emptyDir: {} + name: azure-ip-masq-dir + - name: azure-ip-masq-agent-config-volume + projected: + defaultMode: 420 + sources: + - configMap: + items: + - key: ip-masq-agent + mode: 444 + path: ip-masq-agent + name: azure-ip-masq-agent-config + optional: true + - configMap: + items: + - key: ip-masq-agent-reconciled + mode: 444 + path: ip-masq-agent-reconciled + name: azure-ip-masq-agent-config-reconciled + optional: true + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 2 + type: RollingUpdate diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml new file mode 100644 index 0000000000..7efb9cfa3e --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + ip-masq-agent-reconciled: | + masqLinkLocal: true + nonMasqueradeCIDRs: + - 10.244.0.0/16 + masqLinkLocalIPv6: true +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/managed-by: Eno + component: ip-masq-agent + name: azure-ip-masq-agent-config-reconciled + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml new file mode 100644 index 0000000000..0d9da0b8db --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml @@ -0,0 +1,139 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-algorithm-annotation: "false" + bpf-events-drop-enabled: "true" # exposes drop events to cilium monitor/hubble + bpf-events-policy-verdict-enabled: "true" # exposes policy verdict events to cilium monitor/hubble + bpf-events-trace-enabled: "true" # exposes trace events to cilium monitor/hubble + bpf-filter-priority: "2" + bpf-lb-acceleration: disabled + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode-annotation: "false" + bpf-lb-mode: snat + bpf-lb-sock-terminate-pod-connections: "false" + bpf-lb-sock: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + ces-slice-mode: fcfs + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: default + cni-exclusive: "false" # Cilium takes ownership of /etc/cni/net.d, pods cannot be scheduled with any other cni if cilium is down + cni-log-file: /var/run/cilium/cilium-cni.log + datapath-mode: veth + debug: "false" + direct-routing-skip-unreachable: "false" + disable-cnp-status-updates: "true" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "false" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-bpf-masquerade: "true" + enable-cilium-endpoint-slice: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-endpoint-routes: "true" + enable-experimental-lb: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "false" + enable-hubble: "false" + enable-internal-traffic-policy: "true" + enable-ip-masq-agent: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv4: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-ipv6: "true" + enable-k8s-networkpolicy: "true" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "false" + enable-lb-ipam: "true" + enable-local-node-route: "false" + enable-local-redirect-policy: "false" + enable-masquerade-to-route-source: "false" + enable-metrics: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-runtime-device-detection: "false" + enable-sctp: "false" + enable-session-affinity: "true" + enable-source-ip-verification: "true" + enable-svc-source-range-check: "true" + enable-tcx: "false" # attach endpoint programs with tcx if supported by kernel + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam-cilium-node-update-rate: 15s + ipam: delegated-plugin + ipv4-native-routing-cidr: 10.244.0.0/16 + ipv6-native-routing-cidr: fdd5:a27a:b4bc:99d6::105/64 + k8s-client-burst: "20" + k8s-client-qps: "10" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement-healthz-bind-address: 0.0.0.0:10256 + kube-proxy-replacement: "true" + local-router-ipv4: 169.254.23.0 + local-router-ipv6: 'fe80::' + mesh-auth-enabled: "true" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics: +cilium_bpf_map_pressure + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + monitor-aggregation: medium + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + proxy-connect-timeout: "2" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + remove-cilium-node-taints: "true" + routing-mode: native + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "0" + tofqdns-proxy-response-max-delay: 100ms + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index dccae9cff7..7b8d43fbde 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -120,6 +120,7 @@ var linuxChecksMap = map[string][]check{ podLabelSelector: ciliumLabelSelector, podNamespace: privilegedNamespace, cmd: ciliumStateFileCmd, + containerName: "cilium-agent", }, { name: "cns cache",