diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index d0e88f4f..a2a32414 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -75,3 +75,41 @@ jobs: env: CGO_ENABLED: "0" run: go run gotest.tools/gotestsum@latest -- -count=1 -tags=e2e ./e2e/... -run '^TestConnectivity$' + e2e_windows: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install Windows e2e dependencies + run: | + # Leave room for the installation ISOs, qcow2 and compressed Nix output. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf python3-venv unzip zstd + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + extra-substituters = https://pub-b7e688bb852e422d84f49c2dce8e3461.r2.dev + extra-trusted-public-keys = nylon-cache-1:B87/nHkgelYp2qg2AsYQ+Hmsg99ErIbbvkFCfS7u2WI= + fallback = true + connect-timeout = 5 + download-attempts = 1 + extra-system-features = kvm + - name: Build Windows image + id: windows_image + run: | + nix build github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025 \ + --out-link "$RUNNER_TEMP/windows-image" --fallback + zstd -d "$RUNNER_TEMP/windows-image/nylon-ci-win2025.qcow2.zst" -o "$RUNNER_TEMP/windows.qcow2" + echo "qcow2=$RUNNER_TEMP/windows.qcow2" >> "$GITHUB_OUTPUT" + - name: Run Windows e2e + run: e2e/windows/run.sh '${{ steps.windows_image.outputs.qcow2 }}' + - name: Upload Windows e2e logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: windows-e2e-logs + path: workdir/e2e/windows/**/*.log diff --git a/core/router.go b/core/router.go index aebf19cc..0b3ef987 100644 --- a/core/router.go +++ b/core/router.go @@ -2,6 +2,8 @@ package core import ( "net/netip" + "runtime" + "slices" "time" "github.com/gaissmai/bart" @@ -226,13 +228,32 @@ func (n *Nylon) ComputeSysRouteTable() []netip.Prefix { excludes.RemoveSet(state.MakeSet(n.LocalCfg.UnexcludeIPs)) excludes.AddSet(state.MakeSet(n.LocalCfg.ExcludeIPs)) - final := netipx.IPSetBuilder{} - final.AddSet(state.MakeSet(prefixes)) res, _ := excludes.IPSet() - final.RemoveSet(res) + return systemRoutePrefixes(prefixes, res, runtime.GOOS) +} + +// systemRoutePrefixes subtracts exclusions before routes are reconciled. +// Windows keeps each selected prefix separate: merging adjacent host routes +// creates on-link subnets whose upper address Windows treats as broadcast. +func systemRoutePrefixes(prefixes []netip.Prefix, excluded *netipx.IPSet, goos string) []netip.Prefix { + if goos != "windows" { + included := netipx.IPSetBuilder{} + included.AddSet(state.MakeSet(prefixes)) + included.RemoveSet(excluded) + set, _ := included.IPSet() + return set.Prefixes() + } - res, _ = final.IPSet() - return res.Prefixes() + result := make([]netip.Prefix, 0, len(prefixes)) + for _, prefix := range prefixes { + included := netipx.IPSetBuilder{} + included.AddPrefix(prefix) + included.RemoveSet(excluded) + set, _ := included.IPSet() + result = append(result, set.Prefixes()...) + } + slices.SortFunc(result, func(a, b netip.Prefix) int { return a.Compare(b) }) + return slices.Compact(result) } func (n *Nylon) updatePassiveClient(prefix state.PrefixHealthWrapper, node state.NodeId, passiveHold bool) { diff --git a/core/sys_route_test.go b/core/sys_route_test.go index c3b5b13c..44c18830 100644 --- a/core/sys_route_test.go +++ b/core/sys_route_test.go @@ -2,6 +2,7 @@ package core import ( "net/netip" + "runtime" "slices" "testing" @@ -87,7 +88,22 @@ func TestComputeSysRouteTableCoalescesAdjacentResults(t *testing.T) { }, ) - assert.Equal(t, []netip.Prefix{pfx("10.0.0.0/24")}, sortedPrefixes(n.ComputeSysRouteTable())) + want := []netip.Prefix{pfx("10.0.0.0/24")} + if runtime.GOOS == "windows" { + want = []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")} + } + assert.Equal(t, want, sortedPrefixes(n.ComputeSysRouteTable())) +} + +func TestSystemRoutePrefixesWindowsKeepsAdvertisedPrefixes(t *testing.T) { + hosts := []netip.Prefix{pfx("10.0.0.2/32"), pfx("10.0.0.3/32")} + empty := state.MakeSet(nil) + assert.Equal(t, hosts, systemRoutePrefixes(hosts, empty, "windows")) + assert.Equal(t, []netip.Prefix{pfx("10.0.0.2/31")}, systemRoutePrefixes(hosts, empty, "linux")) + + subnets := []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")} + assert.Equal(t, subnets, systemRoutePrefixes(subnets, empty, "windows")) + assert.Equal(t, []netip.Prefix{pfx("10.0.0.0/24")}, systemRoutePrefixes(subnets, empty, "linux")) } func sysRouteTestNylon(local state.NodeId, centralExcludes, localUnexcludes, localExcludes []netip.Prefix, routes map[netip.Prefix]state.SelRoute) *Nylon { diff --git a/core/sys_windows.go b/core/sys_windows.go index 26941ac4..eaca6a6c 100644 --- a/core/sys_windows.go +++ b/core/sys_windows.go @@ -19,7 +19,7 @@ func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { if addr.Is6() { return Exec(logger, "netsh", "interface", "ipv6", "add", "address", ifName, addr.String()) } - return Exec(logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String()) + return Exec(logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String(), "255.255.255.255") } func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { diff --git a/e2e/windows/README.md b/e2e/windows/README.md new file mode 100644 index 00000000..f9cd2084 --- /dev/null +++ b/e2e/windows/README.md @@ -0,0 +1,30 @@ +# Windows e2e + +Run on x86_64 Ubuntu with Docker, KVM and Go installed: + +```sh +sudo apt-get install qemu-system-x86 qemu-utils ovmf python3-venv unzip curl +# The current user must have read/write access to /dev/kvm and the Docker socket. +e2e/windows/run.sh /absolute/path/to/windows.qcow2 +``` + +The script builds Nylon for Linux and Windows, downloads Wintun, and installs +pywinrm in a local virtualenv. QEMU uses `/usr/bin/qemu-system-x86_64`, +`/usr/bin/qemu-img`, and `/usr/share/OVMF/OVMF_{CODE,VARS}_4M.fd`. + +The image must support UEFI, virtio networking, and WinRM with +`Administrator` / `nylon`. Each VM uses a fresh qcow2 overlay and UEFI variables; +the supplied backing image is unchanged. Guest firewalls are disabled. + +There is one test: `win1 <-> linux <-> win2`. It checks three consecutive ping +replies in both directions between the Windows nodes, and between win1 and Linux. +The Docker network and WinRM ports are fixed; run only one instance per host. +Logs stay in `workdir/e2e/windows`; VMs, overlays, TAPs and containers are removed. + +CI builds `github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025`, +decompresses its output, and passes the qcow2 path to the same script. +CI reads the public Nix cache at +`https://pub-b7e688bb852e422d84f49c2dce8e3461.r2.dev`, trusting the +`nylon-cache-1` public key configured in the workflow. Cache misses or download +failures fall back to local builds. No cache credentials or uploads are needed; +publish the signed final image output to the cache separately. diff --git a/e2e/windows/connectivity.py b/e2e/windows/connectivity.py new file mode 100644 index 00000000..aa50ecd6 --- /dev/null +++ b/e2e/windows/connectivity.py @@ -0,0 +1,186 @@ +"""One fixed topology: win1 <-> linux <-> win2, on an Ubuntu/KVM host.""" + +import functools +import http.server +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import threading +import time +import unittest + +import winrm + + +IMAGE = Path(sys.argv.pop(1)).resolve() +WORK = Path("workdir/e2e/windows").resolve() +NETWORK = "nylon-windows-e2e" +BRIDGE = "nylon-e2e" +LINUX = "nylon-e2e-linux" + + +def run(*args, **kwargs): + return subprocess.run(args, check=True, capture_output=True, text=True, **kwargs).stdout.strip() + + +def host_network(script): + run("docker", "run", "--rm", "--privileged", "--network", "host", + "--entrypoint", "sh", "nylon-debug:latest", "-ec", script) + + +def powershell(port, script): + # Each operation owns its WinRM session and closes the HTTP connection. + session = winrm.Session(f"http://127.0.0.1:{port}/wsman", auth=("Administrator", "nylon"), + transport="ntlm", operation_timeout_sec=10, read_timeout_sec=20) + try: + result = session.run_ps("$ErrorActionPreference = 'Stop';\n" + "[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false);\n" + script) + if result.status_code: + raise RuntimeError(result.std_out.decode() + result.std_err.decode()) + return result.std_out.decode().strip() + finally: + session.protocol.transport.close_session() + + +def wait_for(description, operation, timeout): + deadline = time.monotonic() + timeout + while True: + try: + return operation() + except Exception as error: + if time.monotonic() >= deadline: + raise TimeoutError(f"Waiting for {description}: {error}") from error + time.sleep(1) + + +class WindowsConnectivity(unittest.TestCase): + def start_windows(self, index): + name, port, tap = f"win{index}", 55984 + index, f"nylon-win{index}" + directory = WORK / name + directory.mkdir(exist_ok=True) + disk, variables = directory / "disk.qcow2", directory / "vars.fd" + run("/usr/bin/qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", str(IMAGE), str(disk)) + self.addCleanup(disk.unlink) + shutil.copyfile("/usr/share/OVMF/OVMF_VARS_4M.fd", variables) + self.addCleanup(variables.unlink) + host_network(f"ip tuntap add {tap} mode tap user {os.getuid()}; " + f"ip link set {tap} master {BRIDGE}; ip link set {tap} up") + self.addCleanup(host_network, f"ip link delete {tap}") + log = (directory / "qemu.log").open("w") + self.addCleanup(log.close) + process = subprocess.Popen([ + "/usr/bin/qemu-system-x86_64", "-machine", "q35", "-accel", "kvm", "-cpu", "host", + "-m", "2048", "-smp", "2", "-display", "none", "-monitor", "none", + "-serial", f"file:{directory / 'serial.log'}", + "-drive", "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd", + "-drive", f"if=pflash,format=raw,file={variables}", + "-drive", f"if=ide,format=qcow2,file={disk}", + "-netdev", f"tap,id=data,ifname={tap},script=no,downscript=no", + "-device", f"virtio-net-pci,netdev=data,mac=52:54:00:00:00:0{index}", + "-netdev", f"user,id=mgmt,restrict=on,net=192.168.252.0/24,hostfwd=tcp:127.0.0.1:{port}-:5985", + "-device", "virtio-net-pci,netdev=mgmt,mac=52:54:00:12:34:56", + ], stdout=log, stderr=log) + + def stop(): + process.kill() + process.wait() + + self.addCleanup(stop) + + def configure(): + if process.poll() is not None: + self.fail(f"{name} QEMU exited; see {directory / 'qemu.log'}") + return powershell(port, f""" +$adapter = Get-NetAdapter | Where-Object {{ $_.MacAddress -eq '52-54-00-00-00-0{index}' }} +Set-NetIPInterface -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -Dhcp Disabled +Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false +New-NetIPAddress -InterfaceIndex $adapter.ifIndex -IPAddress '172.30.250.{8 + index * 2}' -PrefixLength 24 | Out-Null +Set-NetFirewallProfile -Profile Domain,Private,Public -Enabled False +[void][IO.Directory]::CreateDirectory('C:\\nylon-e2e') +""") + print(f"Waiting for {name} WinRM and network setup...", flush=True) + wait_for(f"{name} network setup", configure, 300) + return port + + def test_connectivity(self): + run("docker", "network", "create", "--subnet", "172.30.250.0/24", "--ip-range", "172.30.250.128/25", + "--gateway", "172.30.250.1", "-o", f"com.docker.network.bridge.name={BRIDGE}", NETWORK) + self.addCleanup(run, "docker", "network", "rm", NETWORK) + + central = {"routers": [], "graph": ["win1, linux", "linux, win2"], "timestamp": time.time_ns()} + for name, address in [("win1", "10.0.0.1"), ("linux", "10.0.0.2"), ("win2", "10.0.0.3")]: + key = run("docker", "run", "--rm", "--entrypoint", "nylon", "nylon-debug:latest", "genkey") + public = run("docker", "run", "--rm", "-i", "--entrypoint", "nylon", "nylon-debug:latest", + "pubkey", input=key + "\n") + central["routers"].append({"id": name, "pubkey": public, "addresses": [address], + "endpoints": ["172.30.250.11:57175"] if name == "linux" else []}) + local = {"id": name, "key": key, "port": 57175, "interface_name": "nylon0"} + if name != "linux": + local["log_path"] = "C:\\nylon-e2e\\nylon.log" + (WORK / f"{name}.json").write_text(json.dumps(local)) + (WORK / "central.json").write_text(json.dumps(central)) + + run("docker", "run", "-d", "--name", LINUX, "--privileged", "--network", NETWORK, + "--ip", "172.30.250.11", "-v", f"{WORK}:/app/config:ro", "nylon-debug:latest", + "--config", "central.json", "--node", "linux.json") + self.addCleanup(run, "docker", "rm", "-f", LINUX) + + handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(WORK)) + server = http.server.ThreadingHTTPServer(("172.30.250.1", 0), handler) + self.addCleanup(server.server_close) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.shutdown) + url = f"http://172.30.250.1:{server.server_port}" + + ports = {} + for index in (1, 2): + name = f"win{index}" + port = ports[name] = self.start_windows(index) + powershell(port, f""" +foreach ($file in @('nylon.exe', 'wintun.dll', 'central.json', '{name}.json')) {{ + Invoke-WebRequest -UseBasicParsing ('{url}/' + $file) -OutFile ('C:\\nylon-e2e\\' + $file) +}} +# SYSTEM provides UAPI privileges and the scheduled task survives closing WinRM. +$action = New-ScheduledTaskAction -Execute 'C:\\nylon-e2e\\nylon.exe' -WorkingDirectory 'C:\\nylon-e2e' -Argument 'run -v -w --config central.json --node {name}.json' +$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +Register-ScheduledTask -TaskName 'nylon-e2e' -Action $action -Principal $principal | Out-Null +Start-ScheduledTask -TaskName 'nylon-e2e' +""") + + def collect_logs(): + (WORK / "linux.log").write_text(run("docker", "logs", LINUX)) + for name, port in ports.items(): + try: + (WORK / name / "nylon.log").write_text(powershell(port, + "Get-Content 'C:\\nylon-e2e\\nylon.log' -Raw -Encoding UTF8")) + except Exception as error: + print(f"Reading {name} logs: {error}", flush=True) + + self.addCleanup(collect_logs) + + def ping(source, destination, count): + if source == "linux": + output = run("docker", "exec", LINUX, "ping", "-n", "-c", str(count), "-W", "2", destination) + self.assertIn(f"{count} packets transmitted, {count} received", output) + return output + return powershell(ports[source], f""" +$ping = New-Object System.Net.NetworkInformation.Ping +for ($i = 0; $i -lt {count}; $i++) {{ + $reply = $ping.Send('{destination}', 2000) + if ($reply.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) {{ throw $reply.Status }} + $reply.Status +}} +""") + + for source, destination in [("win1", "10.0.0.3"), ("win2", "10.0.0.1"), + ("win1", "10.0.0.2"), ("linux", "10.0.0.1")]: + with self.subTest(source=source, destination=destination): + wait_for(f"{source} -> {destination} convergence", lambda: ping(source, destination, 1), 120) + print(f"{source} -> {destination}: {ping(source, destination, 3)}", flush=True) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/e2e/windows/run.sh b/e2e/windows/run.sh new file mode 100755 index 00000000..00249f56 --- /dev/null +++ b/e2e/windows/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +image=$(realpath "${1:?Usage: e2e/windows/run.sh IMAGE.qcow2}") +cd "$(dirname "$0")/../.." +mkdir -p workdir/e2e/windows + +docker build --target debug -t nylon-debug:latest . +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o workdir/e2e/windows/nylon.exe . +curl -fsSL https://www.wintun.net/builds/wintun-0.14.1.zip -o workdir/e2e/windows/wintun.zip +unzip -p workdir/e2e/windows/wintun.zip wintun/bin/amd64/wintun.dll > workdir/e2e/windows/wintun.dll +python3 -m venv workdir/e2e/windows/venv +workdir/e2e/windows/venv/bin/pip install --quiet pywinrm==0.5.0 +exec workdir/e2e/windows/venv/bin/python e2e/windows/connectivity.py "$image"