From 690dddc22b5b156c7e9ff33a71bbe5e8d35412f1 Mon Sep 17 00:00:00 2001 From: Anson Qian Date: Mon, 1 Dec 2025 17:39:52 +0000 Subject: [PATCH 1/3] add aks integration --- integration/aks/README.md | 36 ++++++ integration/aks/deployment.yaml | 19 +++ integration/aks/service.yaml | 13 ++ integration/aks/setup.py | 206 ++++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+) create mode 100644 integration/aks/README.md create mode 100644 integration/aks/deployment.yaml create mode 100644 integration/aks/service.yaml create mode 100644 integration/aks/setup.py diff --git a/integration/aks/README.md b/integration/aks/README.md new file mode 100644 index 000000000..12970208b --- /dev/null +++ b/integration/aks/README.md @@ -0,0 +1,36 @@ +# AKS Integration + +# Setup Cluster +```bash +RESOURCE_GROUP=cni-test +CLUSTER_NAME=cni-test + +az aks create -l eastus2 \ + --resource-group "${RESOURCE_GROUP}" \ + --cluster-name "${CLUSTER_NAME}" \ + --tier standard \ + --kubernetes-version 1.34.0 \ + --network-plugin none \ + --nodepool-name system \ + --vm-set-type VirtualMachines \ + --node-vm-size Standard_D8ds_v5 \ + --node-count 3 +``` + +# Setup CNI +```bash +python3 setup.py \ + --resource-group "${RESOURCE_GROUP}" \ + --cluster-name "${CLUSTER_NAME}" \ + --ipvlan-prefix-length 28 \ + --boostrap-cni-config +``` + +# Test CNI +```bash +kubectl apply -f deployment.yaml +``` + +```bash +kubectl apply -f service.yaml +``` \ No newline at end of file diff --git a/integration/aks/deployment.yaml b/integration/aks/deployment.yaml new file mode 100644 index 000000000..0a2a7ea6c --- /dev/null +++ b/integration/aks/deployment.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-lb +spec: + selector: + matchLabels: + run: nginx-lb + replicas: 2 + template: + metadata: + labels: + run: nginx-lb + spec: + containers: + - name: nginx-lb + image: nginx + ports: + - containerPort: 80 \ No newline at end of file diff --git a/integration/aks/service.yaml b/integration/aks/service.yaml new file mode 100644 index 000000000..577c34b87 --- /dev/null +++ b/integration/aks/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-svc-lb + labels: + run: nginx-lb +spec: + type: LoadBalancer + ports: + - port: 80 + protocol: TCP + selector: + run: nginx-lb diff --git a/integration/aks/setup.py b/integration/aks/setup.py new file mode 100644 index 000000000..1cba8b8d7 --- /dev/null +++ b/integration/aks/setup.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import subprocess +import ipaddress + +subnet_cache = {} + +def run_az(args, capture=True): + cmd = ["az", *args] + result = subprocess.run(cmd, capture_output=capture, text=True) + if result.returncode != 0: + raise RuntimeError(f"Command failed: {' '.join(cmd)}\n{result.stderr}") + return result.stdout.strip() if capture else "" + + +def subnet_prefix_for(subnet_id): + if not subnet_id: + return "" + if subnet_id in subnet_cache: + return subnet_cache[subnet_id] + subnet = json.loads(run_az([ + "network", "vnet", "subnet", "show", + "--ids", subnet_id, + "-o", "json", + ])) + prefix = subnet.get("addressPrefix") + if not prefix: + prefixes = subnet.get("addressPrefixes") or [] + prefix = prefixes[0] if prefixes else "" + subnet_cache[subnet_id] = prefix + return prefix + + +def scan_node_nics(node_rg): + nic_records = json.loads(run_az([ + "network", "nic", "list", + "--resource-group", node_rg, + "-o", "json", + ]) or "[]") + results = [] + for nic in nic_records: + vm = nic.get("virtualMachine") or {} + vm_id = vm.get("id", "") + vm_name = vm_id.split("/")[-1] if vm_id else "" + ip_configs = [] + for cfg in (nic.get("ipConfigurations") or []): + cfg_name = cfg.get("name", "") + if not (cfg.get("primary") or cfg_name == "ipvlan"): + continue + subnet_id = (cfg.get("subnet") or {}).get("id", "") + ip_configs.append({ + "name": cfg_name, + "primary": bool(cfg.get("primary")), + "ip": cfg.get("privateIPAddress", ""), + "subnet_id": subnet_id, + "subnet_prefix": subnet_prefix_for(subnet_id) if subnet_id else "", + }) + results.append({ + "name": nic.get("name"), + "vm_name": vm_name, + "ip_configs": ip_configs, + }) + return results + + +def boostrap_cni_config(node_rg, nic_name, vm_name, ipvlan_cfg): + if not vm_name or vm_name == "null": + print(f"NIC {nic_name} not attached to a VM; skipping CNI config.") + return + if not ipvlan_cfg: + print(f"NIC {nic_name} missing ipvlan metadata; skipping.") + return + ipvlan_cidr = ipvlan_cfg.get("ip") + if not ipvlan_cidr: + print(f"Unable to read ipvlan IP for NIC {nic_name}; skipping.") + return + subnet_prefix = ipvlan_cfg.get("subnet_prefix") + if not subnet_prefix: + print(f"Unable to read subnet prefix for NIC {subnet_prefix}; skipping.") + return + start, end = derive_range(ipvlan_cidr).split() + + config = { + "cniVersion": "0.3.1", + "name": "ipvlan-eth0", + "type": "ipvlan", + "master": "eth0", + "linkInContainer": False, + "mode": "l3s", + "ipam": { + "type": "host-local", + "ranges": [[{ + "subnet": ipvlan_cidr, + "rangeStart": start, + "rangeEnd": end, + }]], + "routes": [{"dst": "0.0.0.0/0"}], + }, + } + ipvlan_payload = base64.b64encode(json.dumps(config, indent=2).encode()).decode() + print(f"Pushing ipvlan CNI config with subnet {ipvlan_cidr}, rangeStart {start}, rangeEnd {end} to VM {vm_name}...") + scripts = [ + f"echo {ipvlan_payload} | base64 -d | tee /etc/cni/net.d/01-ipvlan-eth0.conf", + f"ip addr replace {ipvlan_cidr} dev eth0", + f"iptables -t nat -A POSTROUTING -s {ipvlan_cidr} ! -d {subnet_prefix} -j MASQUERADE", + ] + run_az([ + "vm", "run-command", "invoke", + "--resource-group", node_rg, + "--name", vm_name, + "--command-id", "RunShellScript", + "--scripts", " & ".join(scripts) + ]) + + +def derive_range(ip_addr): + network = ipaddress.IPv4Network(ip_addr, strict=False) + if network.num_addresses <= 2: + raise ValueError("Prefix too small for usable host range") + start = network.network_address + 1 + end = network.broadcast_address - 1 + return f"{start} {end}" + + +def ensure_ipvlan_ipconfig(node_rg, nic, prefix_length): + nic_name = nic["name"] + ipvlan_cfg = next((cfg for cfg in nic["ip_configs"] if cfg["name"] == "ipvlan"), None) + if ipvlan_cfg: + print(f"Found ipvlan IP config for NIC {nic_name} in {node_rg}...") + return ipvlan_cfg + primary_cfg = next((cfg for cfg in nic["ip_configs"] if cfg.get("primary")), None) + if not primary_cfg: + print(f"Unable to determine primary IP config for NIC {nic_name}; skipping.") + return None + subnet_id = primary_cfg.get("subnet_id") + if not subnet_id: + print(f"Unable to determine subnet for NIC {nic_name}; skipping.") + return None + print(f"Creating ipvlan IP config for NIC {nic_name} in {node_rg}...") + run_az([ + "network", "nic", "ip-config", "create", + "--resource-group", node_rg, + "--nic-name", nic_name, + "--name", "ipvlan", + "--subnet", subnet_id, + "--private-ip-address-version", "IPv4", + "--private-ip-address-prefix-length", str(prefix_length), + ]) + created_cfg = json.loads(run_az([ + "network", "nic", "ip-config", "show", + "--resource-group", node_rg, + "--nic-name", nic_name, + "--name", "ipvlan", + "-o", "json", + ])) + subnet_id = (created_cfg.get("subnet") or {}).get("id", "") + ipvlan_cfg = { + "name": created_cfg.get("name", "ipvlan"), + "primary": bool(created_cfg.get("primary")), + "ip": created_cfg.get("privateIPAddress", ""), + "subnet_id": subnet_id, + "subnet_prefix": subnet_prefix_for(subnet_id), + } + nic["ip_configs"].append(ipvlan_cfg) + print(f"Created ipvlan IP config on {nic_name}.") + return ipvlan_cfg + + +def main(): + parser = argparse.ArgumentParser(description="Sync ipvlan configs for AKS nodes.") + parser.add_argument("--resource-group", required=True) + parser.add_argument("--cluster-name", required=True) + parser.add_argument("--ipvlan-prefix-length", type=int, default=28) + parser.add_argument("--boostrap-cni-config", type=bool, default=False) + args = parser.parse_args() + + node_rg = run_az([ + "aks", "show", + "-g", args.resource_group, + "-n", args.cluster_name, + "--query", "nodeResourceGroup", + "-o", "tsv", + ]) + if not node_rg: + raise RuntimeError(f"Unable to determine node resource group for {args.cluster_name}") + + print(f"Scanning NICs for node resource group {node_rg}.") + nic_views = scan_node_nics(node_rg) + for nic in nic_views: + nic_name = nic["name"] + vm_name = nic["vm_name"] + if not vm_name: + print(f"NIC {nic_name} is detached; skipping CNI config push.") + continue + ipvlan_cfg = ensure_ipvlan_ipconfig(node_rg, nic, args.ipvlan_prefix_length) + if not ipvlan_cfg: + print(f"NIC {nic_name} does not yet have an ipvlan IP config; skipping CNI config push.") + continue + if args.boostrap_cni_config: + boostrap_cni_config(node_rg, nic_name, vm_name, ipvlan_cfg) + +if __name__ == "__main__": + main() From dd2fae35b201d6d927cd6aad91cb94e6b8662dd0 Mon Sep 17 00:00:00 2001 From: Anson Qian Date: Mon, 1 Dec 2025 23:04:12 +0000 Subject: [PATCH 2/3] update test --- integration/aks/README.md | 103 +++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/integration/aks/README.md b/integration/aks/README.md index 12970208b..8aa0628e9 100644 --- a/integration/aks/README.md +++ b/integration/aks/README.md @@ -11,7 +11,6 @@ az aks create -l eastus2 \ --tier standard \ --kubernetes-version 1.34.0 \ --network-plugin none \ - --nodepool-name system \ --vm-set-type VirtualMachines \ --node-vm-size Standard_D8ds_v5 \ --node-count 3 @@ -27,10 +26,112 @@ python3 setup.py \ ``` # Test CNI +Create a deployment with 2 replicas ```bash kubectl apply -f deployment.yaml +deployment.apps/nginx-lb created + +kubectl get pod -l run=nginx-lb -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +nginx-lb-69c48c6986-8l9gh 1/1 Running 0 2m28s 10.224.0.55 aks-default-42863573-vms3 +nginx-lb-69c48c6986-bzgvm 1/1 Running 0 2m28s 10.224.0.28 aks-default-42863573-vms1 ``` +Create a service to expose the deployment ```bash kubectl apply -f service.yaml +service/nginx-svc-lb created + +kubectl get service nginx-svc-lb +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx-svc-lb LoadBalancer 10.0.120.56 68.220.26.204 80:31888/TCP 32s +``` +Test pod-to-pod connectivity +```bash +kubectl exec -it nginx-lb-69c48c6986-bzgvm -- curl 10.224.0.55 + + + +Welcome to nginx! + + + +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ + +``` + +Test service - DNS and cluster ip +```bash +kubectl exec -it nginx-lb-69c48c6986-bzgvm -- curl nginx-svc-lb + + + +Welcome to nginx! + + + +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ + +``` + +Test egress connectivity +```bash +kubectl exec -it nginx-lb-69c48c6986-bzgvm -- curl ifconfig.me +68.220.212.218 +``` + +Test ingress connectivity +```bash + curl 68.220.26.204 + + + +Welcome to nginx! + + + +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ + ``` \ No newline at end of file From 875c13a81f3a0809cdadc95cf698c25cbd1c0448 Mon Sep 17 00:00:00 2001 From: Anson Qian Date: Mon, 1 Dec 2025 23:13:06 +0000 Subject: [PATCH 3/3] fix lint --- integration/aks/deployment.yaml | 8 ++++---- integration/aks/service.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/aks/deployment.yaml b/integration/aks/deployment.yaml index 0a2a7ea6c..ff0a14c31 100644 --- a/integration/aks/deployment.yaml +++ b/integration/aks/deployment.yaml @@ -13,7 +13,7 @@ spec: run: nginx-lb spec: containers: - - name: nginx-lb - image: nginx - ports: - - containerPort: 80 \ No newline at end of file + - name: nginx-lb + image: nginx + ports: + - containerPort: 80 diff --git a/integration/aks/service.yaml b/integration/aks/service.yaml index 577c34b87..440843dee 100644 --- a/integration/aks/service.yaml +++ b/integration/aks/service.yaml @@ -7,7 +7,7 @@ metadata: spec: type: LoadBalancer ports: - - port: 80 - protocol: TCP + - port: 80 + protocol: TCP selector: run: nginx-lb