From 306b405547cfdb98263866a541207ca69aa25d2a Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 25 Nov 2025 09:33:39 -0800 Subject: [PATCH 01/12] feat: cilium ebpf host routing + dualstack --- hack/aks/deploy.mk | 8 + .../cilium/v1.17/ebpf/dualstack/cilium.yaml | 482 ++++++++++++++++++ ...azure-ip-masq-agent-config-reconciled.yaml | 14 + .../ebpf/dualstack/static/cilium-config.yaml | 139 +++++ 4 files changed, 643 insertions(+) create mode 100644 test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml create mode 100644 test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml create mode 100644 test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml diff --git a/hack/aks/deploy.mk b/hack/aks/deploy.mk index 3b02fad9ee..0de61b9940 100644 --- a/hack/aks/deploy.mk +++ b/hack/aks/deploy.mk @@ -26,6 +26,14 @@ deploy-common-ebpf-cilium: @kubectl wait --for=condition=Established crd/ciliumclusterwidenetworkpolicies.cilium.io @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/common/ +deploy-ebpf-dualstack-cilium: deploy-common-ebpf-cilium + @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/static/ + CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_TAG},$${AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY},$${AZURE_IP_MASQ_MERGER_TAG}' < \ + ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/cilium.yaml \ + | kubectl apply -f - + @$(MAKE) wait-for-cilium + deploy-ebpf-overlay-cilium: deploy-common-ebpf-cilium @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/overlay/static/ CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml new file mode 100644 index 0000000000..dfaf899092 --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml @@ -0,0 +1,482 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: cilium + name: cilium + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: cilium + template: + metadata: + annotations: + prometheus.io/port: "9962" + prometheus.io/scrape: "true" + labels: + k8s-app: cilium + kubernetes.azure.com/ebpf-dataplane: cilium + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: + - containerPort: 9962 + hostPort: 9962 + name: prometheus + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 105 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /etc/config + name: azure-ip-masq-dir + readOnly: true + - command: + - ./azure-ip-masq-merger + - -v + - "2" + image: mcr.microsoft.com/containernetworking/azure-ip-masq-merger:v0.0.1 + imagePullPolicy: IfNotPresent + name: azure-ip-masq-merger + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/config/ + name: azure-ip-masq-agent-config-volume + - mountPath: /etc/merged-config/ + name: azure-ip-masq-dir + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - command: + - /install-plugin.sh + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: install-cni-binaries + resources: {} + securityContext: + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: mount-cgroup + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + resources: {} + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: clean-cilium-state + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + appArmorProfile: + type: Unconfined + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/etc/systemd + name: host-etc-systemd + - mountPath: /host/lib/systemd + name: host-lib-systemd + readOnly: true + - mountPath: /host/usr/lib + name: host-usr-lib + readOnly: true + - command: + - /ipv6-hp-bpf + image: mcr.microsoft.com/containernetworking/ipv6-hp-bpf:v0.0.1 + imagePullPolicy: IfNotPresent + name: start-ipv6-hp-bpf + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: ipv6-hp-bpf + - command: + - /bin/bash + - -cx + - | + iptables -t mangle -C FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP + status=$? + set -e + if [ $status -eq 0 ]; then + echo "Skip adding iptables as it already exists" + else + iptables -t mangle -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP + fi + image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + imagePullPolicy: IfNotPresent + name: block-wireserver + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + drop: + - ALL + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/log + type: DirectoryOrCreate + name: ipv6-hp-bpf + - hostPath: + path: /etc/systemd + type: DirectoryOrCreate + name: host-etc-systemd + - hostPath: + path: /lib/systemd + type: DirectoryOrCreate + name: host-lib-systemd + - hostPath: + path: /usr/lib + type: DirectoryOrCreate + name: host-usr-lib + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + secret: + defaultMode: 256 + optional: true + secretName: cilium-clustermesh + - configMap: + defaultMode: 420 + name: cilium-config + name: cilium-config-path + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - configMap: + defaultMode: 420 + name: allowed-iptables-patterns + optional: true + name: iptables-config + - hostPath: + path: /sys/fs/bpf/azure-block-iptables + type: DirectoryOrCreate + name: iptables-block-bpf-map + - emptyDir: {} + name: azure-ip-masq-dir + - name: azure-ip-masq-agent-config-volume + projected: + defaultMode: 420 + sources: + - configMap: + items: + - key: ip-masq-agent + mode: 444 + path: ip-masq-agent + name: azure-ip-masq-agent-config + optional: true + - configMap: + items: + - key: ip-masq-agent-reconciled + mode: 444 + path: ip-masq-agent-reconciled + name: azure-ip-masq-agent-config-reconciled + optional: true + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 2 + type: RollingUpdate diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml new file mode 100644 index 0000000000..7efb9cfa3e --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/azure-ip-masq-agent-config-reconciled.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + ip-masq-agent-reconciled: | + masqLinkLocal: true + nonMasqueradeCIDRs: + - 10.244.0.0/16 + masqLinkLocalIPv6: true +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/managed-by: Eno + component: ip-masq-agent + name: azure-ip-masq-agent-config-reconciled + namespace: kube-system diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml new file mode 100644 index 0000000000..0d9da0b8db --- /dev/null +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/static/cilium-config.yaml @@ -0,0 +1,139 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-algorithm-annotation: "false" + bpf-events-drop-enabled: "true" # exposes drop events to cilium monitor/hubble + bpf-events-policy-verdict-enabled: "true" # exposes policy verdict events to cilium monitor/hubble + bpf-events-trace-enabled: "true" # exposes trace events to cilium monitor/hubble + bpf-filter-priority: "2" + bpf-lb-acceleration: disabled + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode-annotation: "false" + bpf-lb-mode: snat + bpf-lb-sock-terminate-pod-connections: "false" + bpf-lb-sock: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + ces-slice-mode: fcfs + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: default + cni-exclusive: "false" # Cilium takes ownership of /etc/cni/net.d, pods cannot be scheduled with any other cni if cilium is down + cni-log-file: /var/run/cilium/cilium-cni.log + datapath-mode: veth + debug: "false" + direct-routing-skip-unreachable: "false" + disable-cnp-status-updates: "true" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "false" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-bpf-masquerade: "true" + enable-cilium-endpoint-slice: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-endpoint-routes: "true" + enable-experimental-lb: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "false" + enable-hubble: "false" + enable-internal-traffic-policy: "true" + enable-ip-masq-agent: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv4: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-ipv6: "true" + enable-k8s-networkpolicy: "true" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "false" + enable-lb-ipam: "true" + enable-local-node-route: "false" + enable-local-redirect-policy: "false" + enable-masquerade-to-route-source: "false" + enable-metrics: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-runtime-device-detection: "false" + enable-sctp: "false" + enable-session-affinity: "true" + enable-source-ip-verification: "true" + enable-svc-source-range-check: "true" + enable-tcx: "false" # attach endpoint programs with tcx if supported by kernel + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam-cilium-node-update-rate: 15s + ipam: delegated-plugin + ipv4-native-routing-cidr: 10.244.0.0/16 + ipv6-native-routing-cidr: fdd5:a27a:b4bc:99d6::105/64 + k8s-client-burst: "20" + k8s-client-qps: "10" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement-healthz-bind-address: 0.0.0.0:10256 + kube-proxy-replacement: "true" + local-router-ipv4: 169.254.23.0 + local-router-ipv6: 'fe80::' + mesh-auth-enabled: "true" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics: +cilium_bpf_map_pressure + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + monitor-aggregation: medium + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + proxy-connect-timeout: "2" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + remove-cilium-node-taints: "true" + routing-mode: native + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "0" + tofqdns-proxy-response-max-delay: 100ms + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system From 3feeb8a046d09df843beaf9d7aa04605c0750658 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 25 Nov 2025 09:45:53 -0800 Subject: [PATCH 02/12] fix: switch images to env --- .../cilium/v1.17/ebpf/dualstack/cilium.yaml | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml index dfaf899092..cfec665ba8 100644 --- a/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml +++ b/test/integration/manifests/cilium/v1.17/ebpf/dualstack/cilium.yaml @@ -51,7 +51,7 @@ spec: fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 10 @@ -156,7 +156,7 @@ spec: - ./azure-ip-masq-merger - -v - "2" - image: mcr.microsoft.com/containernetworking/azure-ip-masq-merger:v0.0.1 + image: $AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY/azure-ip-masq-merger:$AZURE_IP_MASQ_MERGER_TAG imagePullPolicy: IfNotPresent name: azure-ip-masq-merger resources: {} @@ -170,9 +170,26 @@ spec: dnsPolicy: ClusterFirst hostNetwork: true initContainers: + - command: + - /azure-block-iptables + - -mode=attach + - -overwrite=true + image: $AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY/azure-iptables-monitor:$AZURE_IPTABLES_MONITOR_TAG + imagePullPolicy: IfNotPresent + name: iptables-blocker-init + resources: {} + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /proc + name: hostproc - command: - /install-plugin.sh - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent name: install-cni-binaries resources: {} @@ -200,7 +217,7 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent name: mount-cgroup resources: {} @@ -234,7 +251,7 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites resources: {} @@ -264,7 +281,7 @@ spec: - /bin/bash - -c - -- - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent name: mount-bpf-fs resources: {} @@ -291,7 +308,7 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG imagePullPolicy: IfNotPresent name: clean-cilium-state resources: @@ -325,7 +342,7 @@ spec: readOnly: true - command: - /ipv6-hp-bpf - image: mcr.microsoft.com/containernetworking/ipv6-hp-bpf:v0.0.1 + image: $IPV6_IMAGE_REGISTRY/ipv6-hp-bpf:$IPV6_HP_BPF_VERSION imagePullPolicy: IfNotPresent name: start-ipv6-hp-bpf resources: {} @@ -336,30 +353,6 @@ spec: volumeMounts: - mountPath: /var/log name: ipv6-hp-bpf - - command: - - /bin/bash - - -cx - - | - iptables -t mangle -C FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP - status=$? - set -e - if [ $status -eq 0 ]; then - echo "Skip adding iptables as it already exists" - else - iptables -t mangle -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP - fi - image: mcr.microsoft.com/containernetworking/cilium/cilium:v1.17.0-250211 - imagePullPolicy: IfNotPresent - name: block-wireserver - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - drop: - - ALL - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical From 089a0ff12d7f5db077dbd2d0f59fb3830f002f45 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 25 Nov 2025 10:08:58 -0800 Subject: [PATCH 03/12] feat: add ebpf ds pipeline --- .pipelines/pipeline.yaml | 17 ++++ .../cilium-dualstack-e2e-job-template.yaml | 86 +++++++++++++++++++ .../cilium-dualstack-e2e-step-template.yaml | 48 +++++++++++ 3 files changed, 151 insertions(+) create mode 100644 .pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml create mode 100644 .pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index a249d127ca..e3e7251e58 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -366,6 +366,18 @@ stages: k8sVersion: "" dependsOn: ["test"] + # Cilium EBPF Dualstack Overlay E2E tests + - template: singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml + parameters: + name: "cilium_ebpf_dualstackoverlay_e2e" + displayName: Cilium EBPF on AKS DualStack Overlay + os: linux + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cilbpfdse2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: ["test"] + # Cilium Overlay with hubble E2E tests - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml parameters: @@ -503,6 +515,7 @@ stages: - cilium_nodesubnet_e2e - cilium_overlay_e2e - cilium_ebpf_overlay_e2e + - cilium_ebpf_dualstack_e2e - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e - aks_swift_vnetscale_e2e @@ -543,6 +556,10 @@ stages: name: cilium_ebpf_overlay_e2e clusterName: "cilbpfovere2e" region: $(REGION_AKS_CLUSTER_TEST) + cilium_ebpf_ds_e2e: + name: cilium_ebpf_dualstack_e2e + clusterName: "cilbpfdse2e" + region: $(REGION_AKS_CLUSTER_TEST) cilium_h_overlay_e2e: name: cilium_h_overlay_e2e clusterName: "cilwhleovere2e" diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml new file mode 100644 index 0000000000..f2eaed792e --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml @@ -0,0 +1,86 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + os: "linux" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + osSKU: "AzureLinux" + region: $(REGION_AKS_CLUSTER_TEST) + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cilium EBPF Dualstack Test Suite - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: cilium-dualstack-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + scaleup: 50 + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + dependsOn: ${{ parameters.name }} + datapath: true + dns: true + portforward: true + service: true + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml new file mode 100644 index 0000000000..feaa615a20 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml @@ -0,0 +1,48 @@ +parameters: + name: "" + clusterName: "" + scaleup: "" + + +steps: + - template: ../../templates/setup-environment.yaml + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + kubectl cluster-info + kubectl get pods -Aowide + + # see makefile + export AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY=acnpublic.azurecr.io + export AZURE_IPTABLES_MONITOR_TAG=$(make azure-iptables-monitor-version) + export AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY=acnpublic.azurecr.io + export AZURE_IP_MASQ_MERGER_TAG=$(make azure-ip-masq-merger-version) + make -C ./hack/aks deploy-ebpf-dualstack-cilium + + kubectl get pods -Aowide + name: "installCilium" + displayName: "Install Cilium EBPF on AKS Dualstack Overlay" + + - template: ../../templates/cilium-cli.yaml + + - script: | + CNS=$(make cns-version) IPAM=$(make azure-ipam-version) + kubectl get pods -Aowide + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true + retryCountOnTaskFailure: 3 + name: "aziliumTest" + displayName: "Deploy CNS and Run Azilium E2E on AKS Dualstack Overlay" + + - template: ../../templates/cilium-tests.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${{ parameters.scaleup }} From 009889ab6f81bb3f60ec92791ebce75b9c4ab534 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 25 Nov 2025 10:14:49 -0800 Subject: [PATCH 04/12] fix: stage naming --- .pipelines/pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index e3e7251e58..98cd8e2777 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -369,7 +369,7 @@ stages: # Cilium EBPF Dualstack Overlay E2E tests - template: singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml parameters: - name: "cilium_ebpf_dualstackoverlay_e2e" + name: "cilium_ebpf_ds_e2e" displayName: Cilium EBPF on AKS DualStack Overlay os: linux clusterType: dualstack-byocni-nokubeproxy-up @@ -515,7 +515,7 @@ stages: - cilium_nodesubnet_e2e - cilium_overlay_e2e - cilium_ebpf_overlay_e2e - - cilium_ebpf_dualstack_e2e + - cilium_ebpf_ds_e2e - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e - aks_swift_vnetscale_e2e @@ -557,7 +557,7 @@ stages: clusterName: "cilbpfovere2e" region: $(REGION_AKS_CLUSTER_TEST) cilium_ebpf_ds_e2e: - name: cilium_ebpf_dualstack_e2e + name: cilium_ebpf_ds_e2e clusterName: "cilbpfdse2e" region: $(REGION_AKS_CLUSTER_TEST) cilium_h_overlay_e2e: From 40bc481514a42383e1737096b1ae5b8662c28c44 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 25 Nov 2025 15:10:40 -0800 Subject: [PATCH 05/12] fix: export ipv6 image version and registry --- .../cilium-dualstack-e2e-job-template.yaml | 1 + hack/aks/deploy.mk | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml index f2eaed792e..95cb155373 100644 --- a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-job-template.yaml @@ -42,6 +42,7 @@ stages: GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + IPV6_IMAGE_REGISTRY: acnpublic.azurecr.io pool: name: $(BUILD_POOL_NAME_DEFAULT) jobs: diff --git a/hack/aks/deploy.mk b/hack/aks/deploy.mk index 0de61b9940..d6ec1cbe03 100644 --- a/hack/aks/deploy.mk +++ b/hack/aks/deploy.mk @@ -9,17 +9,19 @@ AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY ?= mcr.microsoft.com/containernetworking AZURE_IP_MASQ_MERGER_TAG ?= v0.0.1-0 # so we can use in envsubst export IPV6_HP_BPF_VERSION +export IPV6_IMAGE_REGISTRY export AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY export AZURE_IPTABLES_MONITOR_TAG export AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY export AZURE_IP_MASQ_MERGER_TAG +export IPV6_HP_BPF_VERSION deploy-common-ebpf-cilium: @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-agent/files/ @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-operator/files/ # set cilium version tag and registry here so they are visible as env vars to envsubst CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ - envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION}' < \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${IPV}' < \ ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/cilium-operator/templates/deployment.yaml \ | kubectl apply -f - @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/common/ciliumclusterwidenetworkpolicies.yaml @@ -29,7 +31,7 @@ deploy-common-ebpf-cilium: deploy-ebpf-dualstack-cilium: deploy-common-ebpf-cilium @kubectl apply -f ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/static/ CILIUM_VERSION_TAG=$(EBPF_CILIUM_VERSION_TAG) CILIUM_IMAGE_REGISTRY=$(EBPF_CILIUM_IMAGE_REGISTRY) \ - envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_TAG},$${AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY},$${AZURE_IP_MASQ_MERGER_TAG}' < \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION},$${IPV6_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_IMAGE_REGISTRY},$${AZURE_IPTABLES_MONITOR_TAG},$${AZURE_IP_MASQ_MERGER_IMAGE_REGISTRY},$${AZURE_IP_MASQ_MERGER_TAG}' < \ ../../test/integration/manifests/cilium/v$(EBPF_CILIUM_DIR)/ebpf/dualstack/cilium.yaml \ | kubectl apply -f - @$(MAKE) wait-for-cilium From a31baefa71ebfaee247c589f7173f013fddc5e39 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 10:28:18 -0800 Subject: [PATCH 06/12] fix: ipv6 cidr --- .pipelines/templates/create-cluster.yaml | 1 + hack/aks/Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index 3b34c16553..e557b73921 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -31,6 +31,7 @@ jobs: echo "Install az cli extension preview" az extension add --name aks-preview az extension update --name aks-preview + export POD_CIDR="10.244.0.0/16,fdd5:a27a:b4bc:99d6::105/64" fi if ! [ -z ${K8S_VERSION} ]; then diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5e1c8f3f9b..f8ea0743cb 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -52,7 +52,7 @@ COMMON_AKS_FIELDS = $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --no-ssh-key \ --os-sku $(OS_SKU) \ $(LTS_ARGS) -POD_CIDR = 192.168.0.0/16 +POD_CIDR ?= 192.168.0.0/16 ##@ Help From 2c17ce93c75dad6faa285bddd0c3d36ed9344c85 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 11:04:19 -0800 Subject: [PATCH 07/12] fix: cilium command --- .pipelines/templates/log.steps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/log.steps.yaml b/.pipelines/templates/log.steps.yaml index 5451fe218b..d8bdfa3eb2 100644 --- a/.pipelines/templates/log.steps.yaml +++ b/.pipelines/templates/log.steps.yaml @@ -190,7 +190,7 @@ steps: echo "Directory created: $(acnLogs)/"$node"_logs/Cilium-output/" file="cilium-endpoint.json" - kubectl exec -i -n kube-system $pod -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file + kubectl exec -i -n kube-system $pod -c cilium-agent -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file echo "Cilium, $file, captured: $(acnLogs)/"$node"_logs/Cilium-output/$file" done fi From e5ce4d7b955caa6efe6d5e8812e47eb218152b47 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 12:15:47 -0800 Subject: [PATCH 08/12] fix: container name --- .pipelines/templates/log-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/log-template.yaml b/.pipelines/templates/log-template.yaml index 17a9d25ab7..9600e0d16d 100644 --- a/.pipelines/templates/log-template.yaml +++ b/.pipelines/templates/log-template.yaml @@ -186,7 +186,7 @@ steps: echo "Directory created: $(acnLogs)/"$node"_logs/Cilium-output/" file="cilium-endpoint.json" - kubectl exec -i -n kube-system $pod -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file + kubectl exec -i -n kube-system $pod -c cilium-agent -- cilium endpoint list -o json > $(acnLogs)/"$node"_logs/Cilium-output/$file echo "Cilium, $file, captured: $(acnLogs)/"$node"_logs/Cilium-output/$file" done fi From 375bd51df7227fd6e3f336f69b84161ad478d4c5 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 13:22:48 -0800 Subject: [PATCH 09/12] fix: container name --- test/validate/linux_validate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index dccae9cff7..7b8d43fbde 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -120,6 +120,7 @@ var linuxChecksMap = map[string][]check{ podLabelSelector: ciliumLabelSelector, podNamespace: privilegedNamespace, cmd: ciliumStateFileCmd, + containerName: "cilium-agent", }, { name: "cns cache", From dbe2112339768307405505c24e4f344b279d566d Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 14:36:12 -0800 Subject: [PATCH 10/12] fix: ipv6 cidr --- .pipelines/templates/create-cluster.yaml | 2 +- hack/aks/Makefile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/create-cluster.yaml b/.pipelines/templates/create-cluster.yaml index e557b73921..549d02d061 100644 --- a/.pipelines/templates/create-cluster.yaml +++ b/.pipelines/templates/create-cluster.yaml @@ -31,7 +31,7 @@ jobs: echo "Install az cli extension preview" az extension add --name aks-preview az extension update --name aks-preview - export POD_CIDR="10.244.0.0/16,fdd5:a27a:b4bc:99d6::105/64" + export POD_CIDRS="10.244.0.0/16,fdd5:a27a:b4bc:99d6::105/64" fi if ! [ -z ${K8S_VERSION} ]; then diff --git a/hack/aks/Makefile b/hack/aks/Makefile index f8ea0743cb..6dcccb5678 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -411,6 +411,7 @@ dualstack-byocni-nokubeproxy-up: rg-up ipv4 ipv6 overlay-net-up ## Brings up a D --network-plugin-mode overlay \ --subscription $(SUB) \ --ip-families ipv4,ipv6 \ + --pod-cidrs "$(POD_CIDRS)" \ --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview,AKSHTTPCustomFeatures=Microsoft.ContainerService/Ubuntu2404Preview \ --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes From de8a21043d2b053b0e45c33f4b834f3d2c60a528 Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Wed, 26 Nov 2025 15:33:11 -0800 Subject: [PATCH 11/12] fix: do not validate statefile --- .../cilium-dualstackoverlay-e2e-step-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 9ef039db57..8a28e0d355 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -55,7 +55,7 @@ steps: - script: | echo "Start Azilium E2E Tests on Overlay Cluster" - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=false INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" From 51af027d99814865d37d885983cf67a20a3e61ad Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu Date: Tue, 2 Dec 2025 09:15:09 -0800 Subject: [PATCH 12/12] fix: use same test as cilium ds overlay --- .../cilium-dualstack-e2e-step-template.yaml | 112 ++++++++++++++++-- ...um-dualstackoverlay-e2e-step-template.yaml | 2 +- 2 files changed, 104 insertions(+), 10 deletions(-) diff --git a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml index feaa615a20..7a19cdc3c7 100644 --- a/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-ebpf/cilium-dualstack-e2e-step-template.yaml @@ -34,15 +34,109 @@ steps: - template: ../../templates/cilium-cli.yaml + - script: | - CNS=$(make cns-version) IPAM=$(make azure-ipam-version) - kubectl get pods -Aowide - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true + echo "Start Azilium E2E Tests on Overlay Cluster" + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" - displayName: "Deploy CNS and Run Azilium E2E on AKS Dualstack Overlay" - - - template: ../../templates/cilium-tests.yaml - parameters: - clusterName: ${{ parameters.clusterName }} - scaleup: ${{ parameters.scaleup }} + displayName: "Run Azilium E2E on AKS Overlay" + + - script: | + kubectl get pods -A + echo "Waiting < 2 minutes for cilium to be ready" + # Ensure Cilium is ready Xm\Xs + cilium status --wait --wait-duration 2m + kubectl get crd -A + retryCountOnTaskFailure: 3 + name: "CiliumStatus" + displayName: "Cilium Status" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + clusterName=${{ parameters.clusterName }} + echo "Restarting nodes" + for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + done + displayName: "Restart Nodes" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + cd test/integration/load + + # Scale Cluster Up/Down to confirm functioning CNS + ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + + cd ../../.. + echo "Validating Node Restart" + make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack + kubectl delete ns load-test + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + + - template: ../../templates/cilium-connectivity-tests.yaml + + - script: | + ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` + echo "##vso[task.setvariable variable=ciliumNamespace]$ns" + retryCountOnTaskFailure: 3 + name: "nsCapture" + displayName: "Capture Connectivity Test Namespace" + + - script: | + set -e + kubectl get po -owide -A + cd test/integration/datapath + echo "Dualstack Overlay Linux datapath IPv6 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true + echo "Dualstack Overlay Linux datapath IPv4 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + retryCountOnTaskFailure: 3 + name: "DualStack_Overlay_Linux_Tests" + displayName: "DualStack Overlay Linux Tests" + + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl get pod -owide -A + cd test/integration/load + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns $(ciliumNamespace) + kubectl get pod -owide -A + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + name: "validatePods" + displayName: "Validate Pods" + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" + + - script: | + cd hack/scripts + chmod +x async-delete-test.sh + ./async-delete-test.sh + if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then + kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' + fi + name: "testAsyncDelete" + displayName: "Verify Async Delete when CNS is down" + + - template: ../../templates/cilium-mtu-check.yaml diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 8a28e0d355..9ef039db57 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -55,7 +55,7 @@ steps: - script: | echo "Start Azilium E2E Tests on Overlay Cluster" - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=false INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay"