From 8e6e6909db3d8a1e39b091ecbdecc81f277327c7 Mon Sep 17 00:00:00 2001 From: Dongruixuan Li Date: Sun, 1 Nov 2026 18:28:40 -0500 Subject: [PATCH 1/4] feat(e2e): support QEMU Windows machines and portable guest operations --- core/router.go | 31 +++- core/sys_route_test.go | 72 +++++++- core/sys_windows.go | 2 +- e2e/README.md | 82 +++++++++ e2e/connectivity_test.go | 67 +++++-- e2e/harness.go | 73 +++++++- e2e/machine/config.go | 89 ++++++++++ e2e/machine/docker.go | 186 ++++++++++++++++++++ e2e/machine/image_linux.go | 138 +++++++++++++++ e2e/machine/image_linux_test.go | 149 ++++++++++++++++ e2e/machine/lifecycle_linux_test.go | 83 +++++++++ e2e/machine/machine.go | 101 +++++++++++ e2e/machine/operations.go | 164 +++++++++++++++++ e2e/machine/operations_test.go | 38 ++++ e2e/machine/qemu_linux.go | 260 +++++++++++++++++++++++++++ e2e/machine/qemu_other.go | 16 ++ e2e/machine/winrm.go | 262 ++++++++++++++++++++++++++++ e2e/machine_test.go | 71 ++++++++ e2e/machines.go | 189 ++++++++++++++++++++ e2e/preflight.go | 162 +++++++++++++++++ go.mod | 19 +- go.sum | 77 ++++++++ 22 files changed, 2301 insertions(+), 30 deletions(-) create mode 100644 e2e/README.md create mode 100644 e2e/machine/config.go create mode 100644 e2e/machine/docker.go create mode 100644 e2e/machine/image_linux.go create mode 100644 e2e/machine/image_linux_test.go create mode 100644 e2e/machine/lifecycle_linux_test.go create mode 100644 e2e/machine/machine.go create mode 100644 e2e/machine/operations.go create mode 100644 e2e/machine/operations_test.go create mode 100644 e2e/machine/qemu_linux.go create mode 100644 e2e/machine/qemu_other.go create mode 100644 e2e/machine/winrm.go create mode 100644 e2e/machine_test.go create mode 100644 e2e/machines.go create mode 100644 e2e/preflight.go diff --git a/core/router.go b/core/router.go index aebf19cc..0b3ef987 100644 --- a/core/router.go +++ b/core/router.go @@ -2,6 +2,8 @@ package core import ( "net/netip" + "runtime" + "slices" "time" "github.com/gaissmai/bart" @@ -226,13 +228,32 @@ func (n *Nylon) ComputeSysRouteTable() []netip.Prefix { excludes.RemoveSet(state.MakeSet(n.LocalCfg.UnexcludeIPs)) excludes.AddSet(state.MakeSet(n.LocalCfg.ExcludeIPs)) - final := netipx.IPSetBuilder{} - final.AddSet(state.MakeSet(prefixes)) res, _ := excludes.IPSet() - final.RemoveSet(res) + return systemRoutePrefixes(prefixes, res, runtime.GOOS) +} + +// systemRoutePrefixes subtracts exclusions before routes are reconciled. +// Windows keeps each selected prefix separate: merging adjacent host routes +// creates on-link subnets whose upper address Windows treats as broadcast. +func systemRoutePrefixes(prefixes []netip.Prefix, excluded *netipx.IPSet, goos string) []netip.Prefix { + if goos != "windows" { + included := netipx.IPSetBuilder{} + included.AddSet(state.MakeSet(prefixes)) + included.RemoveSet(excluded) + set, _ := included.IPSet() + return set.Prefixes() + } - res, _ = final.IPSet() - return res.Prefixes() + result := make([]netip.Prefix, 0, len(prefixes)) + for _, prefix := range prefixes { + included := netipx.IPSetBuilder{} + included.AddPrefix(prefix) + included.RemoveSet(excluded) + set, _ := included.IPSet() + result = append(result, set.Prefixes()...) + } + slices.SortFunc(result, func(a, b netip.Prefix) int { return a.Compare(b) }) + return slices.Compact(result) } func (n *Nylon) updatePassiveClient(prefix state.PrefixHealthWrapper, node state.NodeId, passiveHold bool) { diff --git a/core/sys_route_test.go b/core/sys_route_test.go index c3b5b13c..fc3391f3 100644 --- a/core/sys_route_test.go +++ b/core/sys_route_test.go @@ -2,6 +2,7 @@ package core import ( "net/netip" + "runtime" "slices" "testing" @@ -87,7 +88,76 @@ func TestComputeSysRouteTableCoalescesAdjacentResults(t *testing.T) { }, ) - assert.Equal(t, []netip.Prefix{pfx("10.0.0.0/24")}, sortedPrefixes(n.ComputeSysRouteTable())) + want := []netip.Prefix{pfx("10.0.0.0/24")} + if runtime.GOOS == "windows" { + want = []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")} + } + assert.Equal(t, want, sortedPrefixes(n.ComputeSysRouteTable())) +} + +func TestSystemRoutePrefixesPlatformAggregation(t *testing.T) { + for _, tc := range []struct { + name string + prefixes, excluded []netip.Prefix + merged, separate []netip.Prefix + }{ + { + name: "adjacent hosts", + prefixes: []netip.Prefix{pfx("10.0.0.3/32"), pfx("10.0.0.2/32")}, + merged: []netip.Prefix{pfx("10.0.0.2/31")}, + separate: []netip.Prefix{pfx("10.0.0.2/32"), pfx("10.0.0.3/32")}, + }, + { + name: "four adjacent hosts", + prefixes: []netip.Prefix{pfx("10.0.0.4/32"), pfx("10.0.0.5/32"), pfx("10.0.0.6/32"), pfx("10.0.0.7/32")}, + merged: []netip.Prefix{pfx("10.0.0.4/30")}, + separate: []netip.Prefix{pfx("10.0.0.4/32"), pfx("10.0.0.5/32"), pfx("10.0.0.6/32"), pfx("10.0.0.7/32")}, + }, + { + name: "adjacent subnets", + prefixes: []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")}, + merged: []netip.Prefix{pfx("10.0.0.0/24")}, + separate: []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")}, + }, + { + name: "explicit subnet stays intact", + prefixes: []netip.Prefix{pfx("10.0.0.2/31")}, + merged: []netip.Prefix{pfx("10.0.0.2/31")}, + separate: []netip.Prefix{pfx("10.0.0.2/31")}, + }, + { + name: "IPv6 hosts", + prefixes: []netip.Prefix{pfx("fd00::2/128"), pfx("fd00::3/128")}, + merged: []netip.Prefix{pfx("fd00::2/127")}, + separate: []netip.Prefix{pfx("fd00::2/128"), pfx("fd00::3/128")}, + }, + { + name: "exclusions split an advertised subnet", + prefixes: []netip.Prefix{pfx("10.0.0.0/24")}, + excluded: []netip.Prefix{pfx("10.0.0.64/26")}, + merged: []netip.Prefix{pfx("10.0.0.0/26"), pfx("10.0.0.128/25")}, + separate: []netip.Prefix{pfx("10.0.0.0/26"), pfx("10.0.0.128/25")}, + }, + { + name: "overlap after exclusions is deduplicated", + prefixes: []netip.Prefix{pfx("10.0.0.0/24"), pfx("10.0.0.0/25")}, + excluded: []netip.Prefix{pfx("10.0.0.128/25")}, + merged: []netip.Prefix{pfx("10.0.0.0/25")}, + separate: []netip.Prefix{pfx("10.0.0.0/25")}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + original := slices.Clone(tc.prefixes) + for _, goos := range []string{"linux", "darwin", "windows"} { + want := tc.merged + if goos == "windows" { + want = tc.separate + } + assert.Equal(t, want, systemRoutePrefixes(tc.prefixes, state.MakeSet(tc.excluded), goos), goos) + } + assert.Equal(t, original, tc.prefixes, "selected prefixes must not be mutated") + }) + } } func sysRouteTestNylon(local state.NodeId, centralExcludes, localUnexcludes, localExcludes []netip.Prefix, routes map[netip.Prefix]state.SelRoute) *Nylon { diff --git a/core/sys_windows.go b/core/sys_windows.go index 26941ac4..eaca6a6c 100644 --- a/core/sys_windows.go +++ b/core/sys_windows.go @@ -19,7 +19,7 @@ func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { if addr.Is6() { return Exec(logger, "netsh", "interface", "ipv6", "add", "address", ifName, addr.String()) } - return Exec(logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String()) + return Exec(logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String(), "255.255.255.255") } func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 00000000..2b9051ee --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,82 @@ +# End-to-end machines + +The Linux test runner can launch Nylon in Linux Docker containers or Windows +QEMU VMs. Use `StartMachines`, `machine.RunNylon` and `Harness.Ping` for portable +scenarios; existing tests and auxiliary services can keep using `StartNodes`. +`NodeSpec.OS` overrides `NYLON_E2E_MACHINE` for an individual node. + +```sh +# Existing Linux suite +mise run e2e + +# Windows startup, IPC, ping, process shutdown and three-node forwarding +NYLON_E2E_MACHINE=windows go test -tags=e2e ./e2e \ + -run '^(TestMachines|TestConnectivity)$' -count=1 -parallel=1 -timeout=20m -v + +# Linux endpoints and a Windows relay +NYLON_E2E_MACHINE=mixed go test -tags=e2e ./e2e \ + -run '^TestConnectivity$' -count=1 -parallel=1 -timeout=20m -v + +# Unit tests, without Docker or booting Windows +go test -race ./e2e/machine +``` + +## Setup + +Windows requires a local rootful Linux Docker daemon, `qemu-system-x86_64`, +`qemu-img`, `/dev/net/tun`, writable `/dev/kvm` and OVMF. On Ubuntu install +`qemu-system-x86 qemu-utils ovmf`. Privileged Docker helpers manage TAPs; +QEMU runs as the test user. The image must boot with UEFI, have virtio NIC +and DHCP support, and expose WinRM HTTP/NTLM on port 5985. +The supplied image uses `Administrator` / `nylon` as disposable test credentials. + +Preflight downloads and validates the base image, builds Windows amd64 Nylon, +and prepares Wintun 0.14.1 once per test process, before machines start in +parallel. R2 credentials come from the ignored root `.env`; exported variables +win: `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`. +The default object is `nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst`. +Go handles signing and decompression; AWS CLI and zstd tools are not needed. + +The read-only base is cached in `workdir/e2e/images` using the object filename +without `.zst`. Failed downloads are not published. Clear the cache when +replacing the image; simultaneous test processes sharing this cache are not +supported. Each VM gets a fresh qcow2 backed by that image and its own UEFI vars. +Cleanup removes QEMU, TAPs, overlays and vars while preserving the base and +host/guest logs under `workdir/e2e/{vms,logs}`. + +WinRM uploads the binaries and YAML. Each operation creates an independent +client; each SOAP request owns a temporary encrypted NTLM connection. SYSTEM +scheduled tasks keep Nylon alive after the WinRM shell closes and provide UAPI +permissions. Uploads are checked with SHA256. + +## Networking + +Each scenario has a Docker bridge and `/24`. VM TAPs join that bridge; Docker +allocates dynamic addresses from `.128/25`, while explicit node IPs use `.2`–`.127`. +Each VM also has a restricted management NIC with localhost-only WinRM forwarding; +it cannot carry test traffic through management NAT. The data NIC has no default +gateway. The firewall stays enabled, with allow rules on data and Nylon interfaces. +Docker DNS aliases do not extend to VMs: use service IPs or an explicit resolver. + +Windows keeps selected route prefixes separate; Linux/macOS still merge them. +This avoids adjacent host addresses becoming an on-link subnet whose upper IP +Windows treats as broadcast. Explicitly advertised subnets remain subnet routes. + +## Overrides + +| Variable | Default / purpose | +| --- | --- | +| `NYLON_E2E_WINDOWS_IMAGE` | Existing standalone qcow2; bypass R2 | +| `NYLON_E2E_IMAGE_CACHE` | `workdir/e2e/images` | +| `NYLON_E2E_WINDOWS_BUCKET` / `NYLON_E2E_WINDOWS_OBJECT` | R2 bucket/object above | +| `NYLON_E2E_WINRM_USERNAME` / `NYLON_E2E_WINRM_PASSWORD` | `Administrator` / `nylon` | +| `NYLON_E2E_WINDOWS_MEMORY_MIB` / `NYLON_E2E_WINDOWS_CPUS` | `2048` / `2` per VM | +| `NYLON_E2E_QEMU_ACCEL` / `NYLON_E2E_QEMU_NIC` | `kvm` / `virtio-net-pci`; `tcg` is opt-in | +| `NYLON_E2E_UEFI_CODE` / `NYLON_E2E_UEFI_VARS` | `/usr/share/OVMF/OVMF_{CODE,VARS}_4M.fd` | +| `NYLON_E2E_WINDOWS_BINARY` / `NYLON_E2E_WINTUN_DLL` | Existing Windows amd64 binaries | + +## CI + +CI integration will be added separately once Vault credentials are configured. +The local commands above can be used on a Linux runner with Docker and KVM. +Allow 4 GiB of guest RAM for two Windows nodes or 6 GiB for three. diff --git a/e2e/connectivity_test.go b/e2e/connectivity_test.go index 4600dfc0..73296cf6 100644 --- a/e2e/connectivity_test.go +++ b/e2e/connectivity_test.go @@ -3,9 +3,13 @@ package e2e import ( + "net/netip" + "os" "testing" "time" + "github.com/encodeous/nylon/e2e/machine" + "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" ) @@ -58,27 +62,52 @@ func TestConnectivity(t *testing.T) { node3Cfg := SimpleLocal("node3", node3Key) node3Path := h.WriteConfig(configDir, "node3.yaml", node3Cfg) - // 4. Start Containers in Parallel - h.StartNodes( - NodeSpec{Name: "node1", IP: node1IP, CentralConfigPath: centralPath, NodeConfigPath: node1Path}, - NodeSpec{Name: "node2", IP: node2IP, CentralConfigPath: centralPath, NodeConfigPath: node2Path}, - NodeSpec{Name: "node3", IP: node3IP, CentralConfigPath: centralPath, NodeConfigPath: node3Path}, - ) + // The scenario is identical for Linux, Windows, and a Windows relay between + // Linux endpoints. Only the deployment placement changes. + specs := []NodeSpec{ + {Name: "node1", IP: node1IP, CentralConfigPath: centralPath, NodeConfigPath: node1Path}, + {Name: "node2", IP: node2IP, CentralConfigPath: centralPath, NodeConfigPath: node2Path}, + {Name: "node3", IP: node3IP, CentralConfigPath: centralPath, NodeConfigPath: node3Path}, + } + if os.Getenv("NYLON_E2E_MACHINE") == "mixed" { + specs[0].OS, specs[1].OS, specs[2].OS = machine.Linux, machine.Windows, machine.Linux + } + h.StartMachines(specs...) // 5. Wait for convergence t.Log("Waiting for convergence...") - h.WaitForLog("node3", "installing new route prefix=10.0.0.1/32") - h.WaitForLog("node1", "installing new route prefix=10.0.0.2/31") - - // 6. Test Connectivity - // Ping from node1 to node2's Nylon IP - t.Logf("Pinging %s from node1...", node2NylonIP) - stdout, stderr, err := h.Exec("node3", []string{"ping", "-c", "3", node1NylonIP}) - if err != nil { - h.PrintLogs("node1") - h.PrintLogs("node2") - h.PrintLogs("node3") - t.Fatalf("Ping failed: %v\nStdout: %s\nStderr: %s", err, stdout, stderr) + h.WaitForStatus(t, "node3", func(s *protocol.StatusResponse) bool { + return HasSelectedRoute(s, "10.0.0.1/32", "node2", "node1") + }) + h.WaitForStatus(t, "node1", func(s *protocol.StatusResponse) bool { + return HasSelectedRoute(s, "10.0.0.3/32", "node2", "node3") + }) + // Selected control-plane routes can precede system-route installation. + // Wait for the data plane before asserting consecutive successful replies. + deadline := time.Now().Add(WaitTimeout) + for { + result, err := h.Ping("node3", netip.MustParseAddr(node1NylonIP), machine.PingOptions{Count: 1}) + if err == nil { + break + } + if time.Now().After(deadline) { + t.Fatalf("data plane did not converge: %v\n%s\n%s", err, result.Stdout, result.Stderr) + } + time.Sleep(500 * time.Millisecond) + } + + // Check both multihop directions and a Windows-originated ping in mixed mode. + for _, peer := range []struct{ from, to string }{ + {"node3", node1NylonIP}, {"node2", node1NylonIP}, {"node1", node3NylonIP}, + } { + t.Logf("Pinging %s from %s...", peer.to, peer.from) + result, err := h.Ping(peer.from, netip.MustParseAddr(peer.to), machine.PingOptions{Count: 3}) + if err != nil { + for _, node := range []string{"node1", "node2", "node3"} { + h.PrintLogs(node) + } + t.Fatalf("%s ping to %s: %v\n%s\n%s", peer.from, peer.to, err, result.Stdout, result.Stderr) + } + t.Logf("Ping output:\n%s", result.Stdout) } - t.Logf("Ping output:\n%s", stdout) } diff --git a/e2e/harness.go b/e2e/harness.go index 0cb6fcdd..adce98fc 100644 --- a/e2e/harness.go +++ b/e2e/harness.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/encodeous/nylon/e2e/machine" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "github.com/goccy/go-yaml" @@ -42,6 +43,10 @@ type Harness struct { ctx context.Context Network *testcontainers.DockerNetwork Nodes map[string]testcontainers.Container + Machines map[string]machine.Machine + processes map[string]machine.Process + logCancel context.CancelFunc + logWG sync.WaitGroup LogManager *LogManager ImageName string RootDir string @@ -77,11 +82,15 @@ func NewHarness(t *testing.T) *Harness { } t.Logf("Allocated subnet: %s, gateway: %s", subnet, gateway) + ctx, logCancel := context.WithCancel(ctx) h := &Harness{ t: t, ctx: ctx, Network: newNetwork, Nodes: make(map[string]testcontainers.Container), + Machines: make(map[string]machine.Machine), + processes: make(map[string]machine.Process), + logCancel: logCancel, LogManager: NewLogManager(), RootDir: rootDir, Subnet: subnet, @@ -117,7 +126,9 @@ func createNetwork(ctx context.Context) (*testcontainers.DockerNetwork, string, Driver: "default", Config: []network.IPAMConfig{ { - Subnet: netip.MustParsePrefix(subnet), + Subnet: netip.MustParsePrefix(subnet), + // Dynamic Docker addresses stay clear of static VM addresses. + IPRange: netip.PrefixFrom(netip.MustParseAddr(GetIP(subnet, 128)), 25), Gateway: netip.MustParseAddr(gateway), }, }, @@ -139,6 +150,8 @@ type NodeSpec struct { CentralConfigPath string NodeConfigPath string ExtraArgs []string + // OS is used by StartMachines. Empty selects NYLON_E2E_MACHINE (linux by default). + OS machine.OS } func (h *Harness) StartNodes(specs ...NodeSpec) { @@ -413,10 +426,44 @@ func (h *Harness) StartTrace(nodeName string) { }() } func (h *Harness) Cleanup() { + h.logCancel() + h.logWG.Wait() h.mu.Lock() defer h.mu.Unlock() + if len(h.processes) != 0 { + dir := filepath.Join(h.RootDir, "workdir", "e2e", "logs", h.t.Name()) + if err := os.MkdirAll(dir, 0700); err != nil { + h.t.Logf("create machine log directory: %v", err) + } else { + for name, p := range h.processes { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + r, _, err := p.Poll(cleanupCtx) + cancel() + if err != nil { + h.t.Logf("save machine logs for %s: %v", name, err) + continue + } + for suffix, content := range map[string]string{"stdout": r.Stdout, "stderr": r.Stderr} { + if err := os.WriteFile(filepath.Join(dir, name+"."+suffix+".log"), []byte(content), 0600); err != nil { + h.t.Logf("save machine log: %v", err) + } + } + } + } + } + for name, m := range h.Machines { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + err := m.Close(cleanupCtx) + cancel() + if err != nil { + h.t.Logf("failed to close machine %s: %v", name, err) + } + } for name, c := range h.Nodes { - if err := c.Terminate(h.ctx); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + err := c.Terminate(cleanupCtx) + cancel() + if err != nil { h.t.Logf("failed to terminate container %s: %v", name, err) } } @@ -426,8 +473,20 @@ func (h *Harness) Cleanup() { } func (h *Harness) Exec(nodeName string, cmd []string) (string, string, error) { h.mu.Lock() + m := h.Machines[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() + if m != nil { + if len(cmd) == 0 { + return "", "", fmt.Errorf("empty command") + } + path := cmd[0] + if path == "nylon" && m.Info().OS == machine.Windows { + path = m.Info().WorkDir + `\nylon.exe` + } + r, err := m.Run(h.ctx, machine.Command{Path: path, Args: cmd[1:]}) + return StripAnsi(r.Stdout), StripAnsi(r.Stderr), err + } if !ok { return "", "", fmt.Errorf("node %s not found", nodeName) @@ -488,8 +547,12 @@ func (h *Harness) ExecBackground(nodeName string, cmd []string) *BackgroundExec // GetIP returns the IP address of the node in the test network func (h *Harness) GetIP(nodeName string) (string, error) { h.mu.Lock() + m := h.Machines[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() + if m != nil { + return m.Info().IP, nil + } if !ok { return "", fmt.Errorf("node %s not found", nodeName) } @@ -497,8 +560,14 @@ func (h *Harness) GetIP(nodeName string) (string, error) { } func (h *Harness) PrintLogs(nodeName string) { h.mu.Lock() + p := h.processes[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() + if p != nil { + r, _, err := p.Poll(h.ctx) + h.t.Logf("Logs for %s (error: %v):\n%s\n%s", nodeName, err, r.Stdout, r.Stderr) + return + } if !ok { h.t.Logf("node %s not found for logging", nodeName) return diff --git a/e2e/machine/config.go b/e2e/machine/config.go new file mode 100644 index 00000000..8b09ab89 --- /dev/null +++ b/e2e/machine/config.go @@ -0,0 +1,89 @@ +package machine + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/joho/godotenv" +) + +type WindowsConfig struct { + Name, IP, Subnet, Bridge string + // HostHelperImage supplies iproute2 in a privileged, host-networked Docker + // helper. The Docker daemon must run locally on this Linux QEMU host. + HostHelperImage string + WorkDir string + BaseImage string + CacheDir string + R2Endpoint string + R2AccessKey string + R2SecretKey string + Bucket string + Object string + Username string + Password string + MemoryMiB int + CPUs int + Accelerator string + NICModel string + UEFICode string + UEFIVars string + BootTimeout time.Duration +} + +// WindowsConfigFromEnv reads .env without changing the process environment. +// Exported environment variables take precedence, so the same code works in CI. +func WindowsConfigFromEnv(root string) (WindowsConfig, error) { + values, err := godotenv.Read(filepath.Join(root, ".env")) + if err != nil && !os.IsNotExist(err) { + return WindowsConfig{}, err + } + get := func(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + if value, ok := values[key]; ok { + return value + } + return fallback + } + cfg := WindowsConfig{ + BaseImage: get("NYLON_E2E_WINDOWS_IMAGE", ""), + CacheDir: get("NYLON_E2E_IMAGE_CACHE", filepath.Join(root, "workdir", "e2e", "images")), + WorkDir: filepath.Join(root, "workdir", "e2e", "vms"), + R2Endpoint: get("AWS_ENDPOINT_URL_S3", ""), + R2AccessKey: get("AWS_ACCESS_KEY_ID", ""), + R2SecretKey: get("AWS_SECRET_ACCESS_KEY", ""), + Bucket: get("NYLON_E2E_WINDOWS_BUCKET", "nylon-ci-windows-image"), + Object: get("NYLON_E2E_WINDOWS_OBJECT", "nylon-ci-win2025.qcow2.zst"), + Username: get("NYLON_E2E_WINRM_USERNAME", "Administrator"), + Password: get("NYLON_E2E_WINRM_PASSWORD", "nylon"), + Accelerator: get("NYLON_E2E_QEMU_ACCEL", "kvm"), + NICModel: get("NYLON_E2E_QEMU_NIC", "virtio-net-pci"), + UEFICode: get("NYLON_E2E_UEFI_CODE", "/usr/share/OVMF/OVMF_CODE_4M.fd"), + UEFIVars: get("NYLON_E2E_UEFI_VARS", "/usr/share/OVMF/OVMF_VARS_4M.fd"), + BootTimeout: 5 * time.Minute, + } + for _, field := range []struct { + key, fallback string + dst *int + }{ + {"NYLON_E2E_WINDOWS_MEMORY_MIB", "2048", &cfg.MemoryMiB}, + {"NYLON_E2E_WINDOWS_CPUS", "2", &cfg.CPUs}, + } { + value, err := strconv.Atoi(get(field.key, field.fallback)) + if err != nil || value < 1 { + return WindowsConfig{}, fmt.Errorf("%s must be a positive integer", field.key) + } + *field.dst = value + } + for _, path := range []*string{&cfg.BaseImage, &cfg.CacheDir, &cfg.UEFICode, &cfg.UEFIVars} { + if *path != "" && !filepath.IsAbs(*path) { + *path = filepath.Join(root, *path) + } + } + return cfg, nil +} diff --git a/e2e/machine/docker.go b/e2e/machine/docker.go new file mode 100644 index 00000000..37288530 --- /dev/null +++ b/e2e/machine/docker.go @@ -0,0 +1,186 @@ +package machine + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "strings" + "time" + + "github.com/google/uuid" + "github.com/moby/moby/api/pkg/stdcopy" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" +) + +type DockerConfig struct { + Name string + Image string + Network string + IP string +} + +type Docker struct { + info Info + Container testcontainers.Container + provider *testcontainers.DockerProvider +} + +func NewDocker(ctx context.Context, cfg DockerConfig) (*Docker, error) { + if cfg.Image == "" || cfg.Network == "" { + return nil, fmt.Errorf("Docker machine requires an image and network") + } + if cfg.IP != "" { + if _, err := netip.ParseAddr(cfg.IP); err != nil { + return nil, err + } + } + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: cfg.Image, + Entrypoint: []string{"/bin/sh", "-c", "exec sleep infinity"}, + Networks: []string{cfg.Network}, + NetworkAliases: map[string][]string{cfg.Network: {cfg.Name}}, + HostConfigModifier: func(h *container.HostConfig) { + h.Privileged = true + h.CapAdd = []string{"NET_ADMIN"} + }, + EndpointSettingsModifier: func(endpoints map[string]*network.EndpointSettings) { + if cfg.IP != "" && endpoints[cfg.Network] != nil { + endpoints[cfg.Network].IPAMConfig = &network.EndpointIPAMConfig{IPv4Address: netip.MustParseAddr(cfg.IP)} + } + }, + }, + Started: true, + }) + if err != nil { + if c != nil { + _ = c.Terminate(context.Background()) + } + return nil, err + } + if cfg.IP == "" { + cfg.IP, err = c.ContainerIP(ctx) + if err != nil { + _ = c.Terminate(context.Background()) + return nil, err + } + } + provider, err := testcontainers.NewDockerProvider() + if err != nil { + _ = c.Terminate(context.Background()) + return nil, err + } + return &Docker{info: Info{OS: Linux, IP: cfg.IP, WorkDir: "/app/config"}, Container: c, provider: provider}, nil +} + +func (d *Docker) Info() Info { return d.info } + +func (d *Docker) Run(ctx context.Context, cmd Command) (Result, error) { + if cmd.Path == "" { + return Result{}, fmt.Errorf("empty executable") + } + cli := d.provider.Client() + created, err := cli.ExecCreate(ctx, d.Container.GetContainerID(), client.ExecCreateOptions{ + Cmd: append([]string{cmd.Path}, cmd.Args...), WorkingDir: cmd.Dir, AttachStdout: true, AttachStderr: true, + }) + if err != nil { + return Result{}, err + } + attached, err := cli.ExecAttach(ctx, created.ID, client.ExecAttachOptions{}) + if err != nil { + return Result{}, err + } + defer attached.Close() + stop := context.AfterFunc(ctx, attached.Close) + defer stop() + var stdout, stderr bytes.Buffer + // Drain while the process runs, so output larger than Docker's socket buffer + // cannot deadlock the process before an exit-status inspection. + _, err = stdcopy.StdCopy(&stdout, &stderr, attached.Reader) + r := Result{Stdout: stdout.String(), Stderr: stderr.String()} + if err != nil { + return r, err + } + for { + status, err := cli.ExecInspect(ctx, created.ID, client.ExecInspectOptions{}) + if err != nil { + return r, err + } + if !status.Running { + r.ExitCode = status.ExitCode + return r.check(nil) + } + if err := pause(ctx, 10*time.Millisecond); err != nil { + return r, err + } + } +} + +func (d *Docker) WriteFile(ctx context.Context, path string, data []byte) error { + if _, err := d.Run(ctx, Command{Path: "sh", Args: []string{"-c", `mkdir -p -- "$(dirname -- "$1")"`, "machine", path}}); err != nil { + return err + } + return d.Container.CopyToContainer(ctx, data, path, 0600) +} + +func (d *Docker) Start(ctx context.Context, cmd Command) (Process, error) { + if cmd.Path == "" { + return nil, fmt.Errorf("empty executable") + } + dir := "/tmp/nylon-process-" + uuid.NewString() + if _, err := d.Run(ctx, Command{Path: "mkdir", Args: []string{"-p", dir}}); err != nil { + return nil, err + } + args := []string{shellQuote(cmd.Path)} + for _, a := range cmd.Args { + args = append(args, shellQuote(a)) + } + script := "#!/bin/sh\n" + if cmd.Dir != "" { + script += "cd -- " + shellQuote(cmd.Dir) + " || exit 127\n" + } + script += strings.Join(args, " ") + " >" + shellQuote(dir+"/stdout") + " 2>" + shellQuote(dir+"/stderr") + " &\n" + script += "child=$!\nprintf '%s' \"$child\" >" + shellQuote(dir+"/pid") + "\nwait \"$child\"\ncode=$?\nprintf '%s' \"$code\" >" + shellQuote(dir+"/exit") + "\n" + if err := d.WriteFile(ctx, dir+"/run.sh", []byte(script)); err != nil { + return nil, err + } + _, err := d.Run(ctx, Command{Path: "sh", Args: []string{"-c", `touch "$1/stdout" "$1/stderr" +sh "$1/run.sh" /dev/null 2>&1 & +n=0 +while [ ! -s "$1/pid" ] && [ ! -s "$1/exit" ]; do + n=$((n+1)); [ "$n" -lt 100 ] || exit 1 + sleep 0.05 +done`, "machine", dir}}) + if err != nil { + return nil, err + } + return &dockerProcess{machine: d, dir: dir}, nil +} + +func (d *Docker) Close(ctx context.Context) error { + defer d.provider.Close() + return d.Container.Terminate(ctx) +} + +type dockerProcess struct { + machine *Docker + dir string +} + +func (p *dockerProcess) Poll(ctx context.Context) (Result, bool, error) { + r, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `code=$(cat "$1/exit" 2>/dev/null) +printf '{"stdout":"%s","stderr":"%s","code":%s}' "$(base64 -w0 "$1/stdout")" "$(base64 -w0 "$1/stderr")" "${code:-null}"`, "machine", p.dir}}) + if err != nil { + return Result{}, false, err + } + return processOutput(r.Stdout) +} + +func (p *dockerProcess) Stop(ctx context.Context) error { + _, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `if [ ! -f "$1/exit" ] && [ -f "$1/pid" ]; then kill "$(cat "$1/pid")" 2>/dev/null || true; fi`, "machine", p.dir}}) + return err +} diff --git a/e2e/machine/image_linux.go b/e2e/machine/image_linux.go new file mode 100644 index 00000000..19629482 --- /dev/null +++ b/e2e/machine/image_linux.go @@ -0,0 +1,138 @@ +package machine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/klauspost/compress/zstd" +) + +// PrepareWindowsImage runs during preflight, before any VMs are started. +// A completed image is reused until the cache is explicitly cleared. +func PrepareWindowsImage(ctx context.Context, cfg WindowsConfig) (string, error) { + if cfg.BaseImage != "" { + path, err := filepath.Abs(cfg.BaseImage) + if err != nil { + return "", err + } + return path, validateBaseImage(ctx, path) + } + cacheDir, err := filepath.Abs(cfg.CacheDir) + if err != nil { + return "", err + } + path := filepath.Join(cacheDir, strings.TrimSuffix(filepath.Base(cfg.Object), ".zst")) + if _, err := os.Stat(path); err == nil { + return path, validateBaseImage(ctx, path) + } + if cfg.R2Endpoint == "" || cfg.R2AccessKey == "" || cfg.R2SecretKey == "" { + return "", fmt.Errorf("set AWS_ENDPOINT_URL_S3, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or NYLON_E2E_WINDOWS_IMAGE") + } + u, err := url.Parse(cfg.R2Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") { + return "", fmt.Errorf("invalid R2 endpoint") + } + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return "", err + } + objectURL := u.JoinPath(cfg.Bucket, cfg.Object).String() + client := &http.Client{Timeout: 30 * time.Minute} + response, err := r2Request(ctx, client, cfg, objectURL) + if err != nil { + return "", err + } + defer response.Body.Close() + decoder, err := zstd.NewReader(response.Body) + if err != nil { + return "", err + } + defer decoder.Close() + tmp, err := os.CreateTemp(cacheDir, ".windows-*.qcow2.partial") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, decoder); err != nil { + return "", fmt.Errorf("decompress Windows image: %w", err) + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := validateBaseImage(ctx, tmp.Name()); err != nil { + return "", err + } + if err := os.Chmod(tmp.Name(), 0444); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return "", err + } + return path, nil +} + +func r2Request(ctx context.Context, client *http.Client, cfg WindowsConfig, objectURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, objectURL, nil) + if err != nil { + return nil, err + } + emptyHash := sha256.Sum256(nil) + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(emptyHash[:])) + err = v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: cfg.R2AccessKey, SecretAccessKey: cfg.R2SecretKey}, req, hex.EncodeToString(emptyHash[:]), "s3", "auto", time.Now()) + if err != nil { + return nil, fmt.Errorf("sign R2 request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request Windows image: %w", err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("R2 image: HTTP %d", resp.StatusCode) + } + return resp, nil +} + +func validateBaseImage(ctx context.Context, path string) error { + out, err := exec.CommandContext(ctx, "qemu-img", "info", "--output=json", path).CombinedOutput() + if err != nil { + return fmt.Errorf("inspect base image: %w: %s", err, out) + } + var info struct { + Format string `json:"format"` + Backing string `json:"backing-filename"` + } + if err := json.Unmarshal(out, &info); err != nil { + return err + } + if info.Format != "qcow2" || info.Backing != "" { + return fmt.Errorf("base image must be a standalone qcow2") + } + return nil +} + +// CreateOverlay never writes to the shared backing image. +func CreateOverlay(ctx context.Context, base, overlay string) error { + absolute, err := filepath.Abs(base) + if err != nil { + return err + } + out, err := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", absolute, overlay).CombinedOutput() + if err != nil { + return fmt.Errorf("create qcow2 overlay: %w: %s", err, out) + } + return nil +} diff --git a/e2e/machine/image_linux_test.go b/e2e/machine/image_linux_test.go new file mode 100644 index 00000000..fd8785d0 --- /dev/null +++ b/e2e/machine/image_linux_test.go @@ -0,0 +1,149 @@ +package machine + +import ( + "crypto/sha256" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/klauspost/compress/zstd" +) + +func smallBase(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("qemu-img"); err != nil { + t.Skip("qemu-img is not installed") + } + path := filepath.Join(t.TempDir(), "base.qcow2") + if out, err := exec.Command("qemu-img", "create", "-f", "qcow2", path, "8M").CombinedOutput(); err != nil { + t.Fatalf("create base: %v: %s", err, out) + } + return path +} + +func TestOverlaysIsolateWritesAndPreserveBase(t *testing.T) { + base := smallBase(t) + if _, err := exec.LookPath("qemu-io"); err != nil { + t.Skip("qemu-io is not installed") + } + before, err := os.ReadFile(base) + if err != nil { + t.Fatal(err) + } + baseHash := sha256.Sum256(before) + paths := []string{filepath.Join(t.TempDir(), "first.qcow2"), filepath.Join(t.TempDir(), "second.qcow2")} + for _, path := range paths { + if err := CreateOverlay(t.Context(), base, path); err != nil { + t.Fatal(err) + } + out, err := exec.Command("qemu-img", "info", "--output=json", path).CombinedOutput() + if err != nil { + t.Fatal(err) + } + var info struct { + Backing string `json:"full-backing-filename"` + Format string `json:"backing-filename-format"` + } + if err := json.Unmarshal(out, &info); err != nil { + t.Fatal(err) + } + if info.Backing != base || info.Format != "qcow2" { + t.Fatalf("unexpected backing: %+v", info) + } + } + if out, err := exec.Command("qemu-io", "-f", "qcow2", "-c", "write -P 0x42 0 512", paths[0]).CombinedOutput(); err != nil { + t.Fatalf("write overlay: %v: %s", err, out) + } + if out, err := exec.Command("qemu-io", "-r", "-f", "qcow2", "-c", "read -P 0 0 512", paths[1]).CombinedOutput(); err != nil { + t.Fatalf("second overlay changed: %v: %s", err, out) + } + after, err := os.ReadFile(base) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(after) != baseHash { + t.Fatal("backing image was modified") + } +} + +func TestR2CacheRejectsIncompleteDownloadAndReusesImage(t *testing.T) { + base := smallBase(t) + data, err := os.ReadFile(base) + if err != nil { + t.Fatal(err) + } + encoder, err := zstd.NewWriter(nil) + if err != nil { + t.Fatal(err) + } + defer encoder.Close() + compressed := encoder.EncodeAll(data, nil) + var gets atomic.Int32 + var corrupt atomic.Bool + corrupt.Store(true) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 ") || r.Header.Get("X-Amz-Content-Sha256") == "" { + t.Error("unsigned R2 request") + w.WriteHeader(403) + return + } + if r.URL.Path != "/bucket/image.qcow2.zst" { + t.Errorf("unexpected object: %s", r.URL.Path) + } + if r.Method != http.MethodGet { + t.Errorf("unexpected method: %s", r.Method) + } + gets.Add(1) + if corrupt.Load() { + _, _ = w.Write([]byte("interrupted image")) + return + } + _, _ = w.Write(compressed) + })) + defer server.Close() + cfg := WindowsConfig{R2Endpoint: server.URL, R2AccessKey: "test-key", R2SecretKey: "test-secret", Bucket: "bucket", Object: "image.qcow2.zst", CacheDir: t.TempDir()} + if _, err := PrepareWindowsImage(t.Context(), cfg); err == nil { + t.Fatal("corrupt compressed image accepted") + } + files, err := filepath.Glob(filepath.Join(cfg.CacheDir, "*.qcow2")) + if err != nil { + t.Fatal(err) + } + if len(files) != 0 { + t.Fatal("incomplete image published to cache") + } + corrupt.Store(false) + path, err := PrepareWindowsImage(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + // Warm-cache preflight needs no R2 credentials or network request. + cfg.R2Endpoint, cfg.R2AccessKey, cfg.R2SecretKey = "", "", "" + cached, err := PrepareWindowsImage(t.Context(), cfg) + if err != nil || cached != path { + t.Fatalf("cache reuse: %s, %v", cached, err) + } + if gets.Load() != 2 { + t.Fatalf("got %d downloads; expected one failed and one successful", gets.Load()) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0222 != 0 { + t.Fatal("cached backing image is writable") + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(got) != sha256.Sum256(data) { + t.Fatal("cached image content changed") + } +} diff --git a/e2e/machine/lifecycle_linux_test.go b/e2e/machine/lifecycle_linux_test.go new file mode 100644 index 00000000..9c79b641 --- /dev/null +++ b/e2e/machine/lifecycle_linux_test.go @@ -0,0 +1,83 @@ +//go:build linux + +package machine + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestWinRMHonorsCanceledContext(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + w.(http.Flusher).Flush() + close(started) + <-release + })) + defer server.Close() + _, portText, _ := net.SplitHostPort(server.Listener.Addr().String()) + port, _ := strconv.Atoi(portText) + w := QEMUWindows{port: port, cfg: WindowsConfig{Username: "test", Password: "test"}} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { _, err := w.runPS(ctx, "'hello'", ""); done <- err }() + <-started + cancel() + select { + case err := <-done: + close(release) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled operation, got %v", err) + } + case <-time.After(time.Second): + t.Errorf("runPS still blocked after context expired: ctx.Err()=%v", ctx.Err()) + close(release) + t.Logf("after HTTP response body was released: %v", <-done) + } +} + +func TestCloseCanRetryAfterCancellation(t *testing.T) { + dir := t.TempDir() + disk := filepath.Join(dir, "disk.qcow2") + vars := filepath.Join(dir, "uefi-vars.fd") + for _, p := range []string{disk, vars} { + if err := os.WriteFile(p, []byte("test"), 0600); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("sleep", "30") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + reaped := make(chan struct{}) + done := make(chan struct{}) + w := QEMUWindows{dir: dir, cmd: cmd, done: done} + // Model an already cancelled harness budget and a reaper not scheduled yet. + go func() { _ = cmd.Wait(); close(reaped); <-done }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + first := w.Close(ctx) + <-reaped + close(done) + second := w.Close(context.Background()) + if second != nil { + t.Fatalf("cleanup retry: %v (first: %v)", second, first) + } + for _, path := range []string{disk, vars} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("cleanup did not remove %s: %v", path, err) + } + } +} diff --git a/e2e/machine/machine.go b/e2e/machine/machine.go new file mode 100644 index 00000000..63dbda39 --- /dev/null +++ b/e2e/machine/machine.go @@ -0,0 +1,101 @@ +// Package machine provides the machines and guest operations used by e2e tests. +// The test runner lives on Linux; a guest may be a Linux container or Windows VM. +package machine + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +type OS string + +const ( + Linux OS = "linux" + Windows OS = "windows" +) + +type Info struct { + OS OS + IP string // Address on the test data network, never the management NIC. + WorkDir string +} + +// Command passes arguments separately. Shell syntax is only interpreted when +// the executable is explicitly a shell. +type Command struct { + Path string + Args []string + Dir string +} + +type Result struct { + Stdout string + Stderr string + ExitCode int +} + +func (r Result) check(err error) (Result, error) { + if err != nil { + return r, err + } + if r.ExitCode != 0 { + return r, fmt.Errorf("guest command exited with code %d: %s\n%s", r.ExitCode, r.Stdout, r.Stderr) + } + return r, nil +} + +type Machine interface { + Info() Info + Run(context.Context, Command) (Result, error) + WriteFile(context.Context, string, []byte) error + Start(context.Context, Command) (Process, error) + Close(context.Context) error +} + +// Process survives the command session that launched it. Output is a cumulative +// snapshot; callers can subscribe by retaining their last byte offset. +type Process interface { + Poll(context.Context) (output Result, exited bool, err error) + Stop(context.Context) error +} + +func guestPath(m Machine, name string) string { + sep := "/" + if m.Info().OS == Windows { + sep = `\` + } + return strings.TrimRight(m.Info().WorkDir, `/\`) + sep + name +} + +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } +func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } + +func pause(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// Both backends return base64 output in JSON so arbitrary bytes survive transport. +func processOutput(data string) (Result, bool, error) { + var snapshot struct { + Stdout, Stderr []byte + Code *int + } + if err := json.Unmarshal([]byte(data), &snapshot); err != nil { + return Result{}, false, fmt.Errorf("decode process output: %w", err) + } + result := Result{Stdout: string(snapshot.Stdout), Stderr: string(snapshot.Stderr)} + if snapshot.Code != nil { + result.ExitCode = *snapshot.Code + } + return result, snapshot.Code != nil, nil +} diff --git a/e2e/machine/operations.go b/e2e/machine/operations.go new file mode 100644 index 00000000..d91dd8d2 --- /dev/null +++ b/e2e/machine/operations.go @@ -0,0 +1,164 @@ +package machine + +import ( + "context" + "fmt" + "math" + "net/netip" + "os" + "strconv" + "strings" + "time" + + "github.com/encodeous/nylon/state" + "github.com/goccy/go-yaml" +) + +type NylonOptions struct { + // Host paths prepared by preflight for Windows; Linux uses its Docker image. + Binary string + WintunDLL string + ExtraArgs []string + StartupTimeout time.Duration +} + +// RunNylon installs config and optional artifacts, launches Nylon, and waits for +// initialization. Routing convergence is left to the scenario's assertions. +func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local state.LocalCfg, opts NylonOptions) (_ Process, err error) { + // Bound setup as well as initialization: an interrupted WinRM upload must + // not hang a test until go test's hard timeout (which bypasses cleanup). + ctx, cancelSetup := context.WithTimeout(ctx, 5*time.Minute) + defer cancelSetup() + centralData, err := yaml.Marshal(central) + if err != nil { + return nil, err + } + localData, err := yaml.Marshal(local) + if err != nil { + return nil, err + } + for name, data := range map[string][]byte{"central.yaml": centralData, "node.yaml": localData} { + if err := m.WriteFile(ctx, guestPath(m, name), data); err != nil { + return nil, fmt.Errorf("write %s: %w", name, err) + } + } + binary := "/usr/local/bin/nylon" + if m.Info().OS == Windows { + binary = guestPath(m, "nylon.exe") + if opts.Binary == "" || opts.WintunDLL == "" { + return nil, fmt.Errorf("Windows requires NylonOptions.Binary and WintunDLL") + } + for name, source := range map[string]string{"nylon.exe": opts.Binary, "wintun.dll": opts.WintunDLL} { + if err := upload(ctx, m, source, guestPath(m, name)); err != nil { + return nil, err + } + } + } + args := []string{"run", "-v", "-w", "--dbg-trace-tc", "--config", guestPath(m, "central.yaml"), "--node", guestPath(m, "node.yaml")} + args = append(args, opts.ExtraArgs...) + p, err := m.Start(ctx, Command{Path: binary, Args: args, Dir: m.Info().WorkDir}) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _ = p.Stop(cleanupCtx) + } + }() + timeout := opts.StartupTimeout + if timeout == 0 { + timeout = 30 * time.Second + } + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var output Result + for { + var exited bool + output, exited, err = p.Poll(waitCtx) + if err != nil { + return nil, fmt.Errorf("read Nylon startup output: %w", err) + } + if exited { + return nil, fmt.Errorf("Nylon exited during startup (%d): %s\n%s", output.ExitCode, output.Stdout, output.Stderr) + } + if strings.Contains(output.Stdout+output.Stderr, "Nylon has been initialized") { + if m.Info().OS == Windows && !local.NoTun { + name := local.InterfaceName + if name == "" { + name = "nylon" + } + // Wintun appears after the machine's data-NIC firewall setup. + // Allow the scenario's inner packets on this disposable interface. + script := "New-NetFirewallRule -DisplayName 'Nylon e2e overlay inbound' -Direction Inbound -InterfaceAlias " + psQuote(name) + " -Action Allow -Profile Any | Out-Null\n" + + "New-NetFirewallRule -DisplayName 'Nylon e2e overlay outbound' -Direction Outbound -InterfaceAlias " + psQuote(name) + " -Action Allow -Profile Any | Out-Null" + if _, err := m.Run(waitCtx, Command{Path: "powershell.exe", Args: []string{"-NoProfile", "-NonInteractive", "-Command", script}}); err != nil { + return nil, fmt.Errorf("configure Windows overlay firewall: %w", err) + } + } + return p, nil + } + if err := pause(waitCtx, 250*time.Millisecond); err != nil { + return nil, fmt.Errorf("waiting for Nylon initialization: %w\n%s\n%s", err, output.Stdout, output.Stderr) + } + } +} + +func upload(ctx context.Context, m Machine, src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return err + } + if err := m.WriteFile(ctx, dst, data); err != nil { + return fmt.Errorf("upload %s: %w", dst, err) + } + return nil +} + +type PingOptions struct { + Count int + Timeout time.Duration // Per reply. +} + +// Ping executes the guest's ping against an IP, exercising its real system +// routing and TUN. Failure is determined by replies, including on Windows where +// ping.exe may otherwise return success for a destination-unreachable reply. +func Ping(ctx context.Context, m Machine, addr netip.Addr, opts PingOptions) (Result, error) { + if !addr.IsValid() { + return Result{}, fmt.Errorf("invalid ping address") + } + if opts.Count == 0 { + opts.Count = 3 + } + if opts.Timeout == 0 { + opts.Timeout = 2 * time.Second + } + if opts.Count < 1 || opts.Timeout < 0 { + return Result{}, fmt.Errorf("invalid ping options") + } + // Include intervals and command startup, while respecting an earlier caller deadline. + ctx, cancel := context.WithTimeout(ctx, time.Duration(opts.Count)*(opts.Timeout+time.Second)+15*time.Second) + defer cancel() + if m.Info().OS == Windows { + // .NET Ping is independent of the language of the Windows installation and + // gives an explicit IPStatus for each reply. It uses the guest network stack. + script := fmt.Sprintf(`$p = New-Object System.Net.NetworkInformation.Ping +$failed = $false +try { + for ($i = 0; $i -lt %d; $i++) { + $r = $p.Send(%s, %d) + Write-Output ($r.Status.ToString() + ' ' + $r.RoundtripTime + 'ms') + if ($r.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) { $failed = $true } + } +} finally { $p.Dispose() } +if ($failed) { exit 1 }`, opts.Count, psQuote(addr.String()), (opts.Timeout+time.Millisecond-1)/time.Millisecond) + return m.Run(ctx, Command{Path: "powershell.exe", Args: []string{"-NoProfile", "-NonInteractive", "-Command", script}}) + } + // With both count and deadline, iputils exits nonzero unless all requested + // replies arrive. This matches the Windows IPStatus checks above. + deadline := int(math.Ceil(float64(opts.Count) * (opts.Timeout + time.Second).Seconds())) + args := []string{"-n", "-c", strconv.Itoa(opts.Count), "-W", strconv.FormatFloat(opts.Timeout.Seconds(), 'f', 3, 64), "-w", strconv.Itoa(deadline)} + args = append(args, addr.String()) + return m.Run(ctx, Command{Path: "ping", Args: args}) +} diff --git a/e2e/machine/operations_test.go b/e2e/machine/operations_test.go new file mode 100644 index 00000000..b1ffc6a9 --- /dev/null +++ b/e2e/machine/operations_test.go @@ -0,0 +1,38 @@ +package machine + +import ( + "context" + "strings" + "testing" + + "github.com/encodeous/nylon/state" +) + +type failedProcess struct { + stopped bool +} + +func (p *failedProcess) Poll(context.Context) (Result, bool, error) { + return Result{Stderr: "Nylon has been initialized\nprocess crashed", ExitCode: 42}, true, nil +} +func (p *failedProcess) Stop(context.Context) error { p.stopped = true; return nil } + +type testMachine struct{ process Process } + +func (m *testMachine) Info() Info { return Info{OS: Linux, WorkDir: "/app/config"} } +func (m *testMachine) Run(context.Context, Command) (Result, error) { return Result{}, nil } +func (m *testMachine) Start(context.Context, Command) (Process, error) { return m.process, nil } +func (m *testMachine) WriteFile(context.Context, string, []byte) error { return nil } +func (m *testMachine) Close(context.Context) error { return nil } + +func TestRunNylonRejectsExitedProcessEvenWithInitializationLog(t *testing.T) { + p := &failedProcess{} + m := &testMachine{process: p} + _, err := RunNylon(t.Context(), m, state.CentralCfg{}, state.LocalCfg{}, NylonOptions{}) + if err == nil || !strings.Contains(err.Error(), "42") { + t.Fatalf("expected startup exit failure, got %v", err) + } + if !p.stopped { + t.Fatal("failed startup process was not cleaned up") + } +} diff --git a/e2e/machine/qemu_linux.go b/e2e/machine/qemu_linux.go new file mode 100644 index 00000000..68afe666 --- /dev/null +++ b/e2e/machine/qemu_linux.go @@ -0,0 +1,260 @@ +package machine + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +type QEMUWindows struct { + info Info + port int + cfg WindowsConfig + dir, tap string + cmd *exec.Cmd + done chan struct{} + waitErr error + log *os.File +} + +// NewWindows boots an isolated UEFI guest. Its TAP joins the test bridge; a +// separate user-mode management NIC forwards WinRM to localhost. Neither test +// routes nor link failures can strand the management connection. +// BaseImage must contain the standalone qcow2 prepared by preflight. +func NewWindows(ctx context.Context, cfg WindowsConfig) (_ *QEMUWindows, err error) { + addr, err := netip.ParseAddr(cfg.IP) + if err != nil || !addr.Is4() { + return nil, fmt.Errorf("Windows machine requires a valid IPv4 data address") + } + prefix, err := netip.ParsePrefix(cfg.Subnet) + if err != nil || !prefix.Contains(addr) { + return nil, fmt.Errorf("Windows data IP must belong to the test subnet") + } + if cfg.Bridge == "" || cfg.HostHelperImage == "" { + return nil, fmt.Errorf("Windows requires a local Docker bridge and host helper image") + } + if cfg.Accelerator != "kvm" && cfg.Accelerator != "tcg" { + return nil, fmt.Errorf("QEMU accelerator must be kvm or tcg") + } + if cfg.MemoryMiB < 1 || cfg.CPUs < 1 || cfg.BootTimeout <= 0 { + return nil, fmt.Errorf("invalid Windows VM resources or boot timeout") + } + if _, err := exec.LookPath("qemu-system-x86_64"); err != nil { + return nil, err + } + if cfg.Accelerator == "kvm" { + kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("KVM unavailable (use NYLON_E2E_QEMU_ACCEL=tcg explicitly for emulation): %w", err) + } + kvm.Close() + } + if cfg.BaseImage == "" { + return nil, fmt.Errorf("Windows base image must be prepared by preflight") + } + if err := os.MkdirAll(cfg.WorkDir, 0700); err != nil { + return nil, err + } + dir, err := os.MkdirTemp(cfg.WorkDir, "windows-") + if err != nil { + return nil, err + } + w := &QEMUWindows{cfg: cfg, dir: dir, info: Info{OS: Windows, IP: cfg.IP, WorkDir: `C:\nylon-e2e`}} + defer func() { + if err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + err = errors.Join(err, w.Close(cleanupCtx)) + } + }() + overlay := filepath.Join(dir, "disk.qcow2") + if err := CreateOverlay(ctx, cfg.BaseImage, overlay); err != nil { + return nil, err + } + vars, err := os.ReadFile(cfg.UEFIVars) + if err != nil { + return nil, fmt.Errorf("read UEFI variable template: %w", err) + } + varsPath := filepath.Join(dir, "uefi-vars.fd") + if err := os.WriteFile(varsPath, vars, 0600); err != nil { + return nil, err + } + if _, err := os.Stat(cfg.UEFICode); err != nil { + return nil, fmt.Errorf("read UEFI firmware: %w", err) + } + var random [5]byte + if _, err := rand.Read(random[:]); err != nil { + return nil, err + } + w.tap = "ny" + hex.EncodeToString(random[:]) // Linux interface names <= 15 bytes. + macSuffix := fmt.Sprintf("%02x:%02x:%02x", random[0], random[1], random[2]) + dataMAC, mgmtMAC := "52:54:00:"+macSuffix, "52:54:01:"+macSuffix + if err := w.hostNetwork(ctx, `ip tuntap add dev "$1" mode tap user "$3" && ip link set dev "$1" master "$2" && ip link set dev "$1" up`, w.tap, cfg.Bridge, strconv.Itoa(os.Getuid())); err != nil { + return nil, err + } + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return nil, err + } + w.port = listener.Addr().(*net.TCPAddr).Port + listener.Close() + w.log, err = os.OpenFile(filepath.Join(dir, "qemu.log"), os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return nil, err + } + cpu := "max" + if cfg.Accelerator == "kvm" { + cpu = "host" + } + args := []string{ + "-name", cfg.Name, "-machine", "q35", "-accel", cfg.Accelerator, "-cpu", cpu, + "-m", strconv.Itoa(cfg.MemoryMiB), "-smp", strconv.Itoa(cfg.CPUs), + "-display", "none", "-monitor", "none", "-serial", "file:" + qemuValue(filepath.Join(dir, "serial.log")), + "-drive", "if=pflash,format=raw,readonly=on,file=" + qemuValue(cfg.UEFICode), + "-drive", "if=pflash,format=raw,file=" + qemuValue(varsPath), + "-drive", "if=ide,format=qcow2,file=" + qemuValue(overlay), + "-netdev", "tap,id=data,ifname=" + w.tap + ",script=no,downscript=no", + "-device", cfg.NICModel + ",netdev=data,mac=" + dataMAC, + "-netdev", fmt.Sprintf("user,id=mgmt,restrict=on,net=192.168.252.0/24,hostfwd=tcp:127.0.0.1:%d-:5985", w.port), + "-device", cfg.NICModel + ",netdev=mgmt,mac=" + mgmtMAC, + } + w.cmd = exec.Command("qemu-system-x86_64", args...) + // Do not leave VMs running if go test is killed or reaches its hard timeout. + w.cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + w.cmd.Stdout, w.cmd.Stderr = w.log, w.log + if err := w.cmd.Start(); err != nil { + return nil, err + } + w.done = make(chan struct{}) + go func() { w.waitErr = w.cmd.Wait(); close(w.done) }() + bootCtx, cancel := context.WithTimeout(ctx, cfg.BootTimeout) + defer cancel() + var lastErr error + for { + select { + case <-w.done: + return nil, fmt.Errorf("QEMU exited during boot: %v (see %s)", w.waitErr, filepath.Join(dir, "qemu.log")) + default: + } + probeCtx, probeCancel := context.WithTimeout(bootCtx, 20*time.Second) + _, lastErr = w.runPS(probeCtx, "[Console]::Out.Write('ready')", "") + probeCancel() + if lastErr == nil { + break + } + if err := pause(bootCtx, time.Second); err != nil { + return nil, fmt.Errorf("Windows WinRM readiness: %w; last error: %v (see %s)", err, lastErr, filepath.Join(dir, "qemu.log")) + } + } + // Select the data adapter by MAC, never by an unstable Windows adapter name. + // It has no default gateway, leaving management replies on the management NIC. + mac := strings.ReplaceAll(dataMAC, ":", "") + script := `$adapter = Get-NetAdapter | Where-Object { ($_.MacAddress -replace '[:-]', '') -eq ` + psQuote(mac) + ` } +if (!$adapter) { throw 'QEMU test data adapter not found' } +$idx = $adapter.ifIndex +Set-NetIPInterface -InterfaceIndex $idx -AddressFamily IPv4 -Dhcp Disabled +Get-NetIPAddress -InterfaceIndex $idx -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false -ErrorAction SilentlyContinue +New-NetIPAddress -InterfaceIndex $idx -IPAddress ` + psQuote(cfg.IP) + ` -PrefixLength ` + strconv.Itoa(prefix.Bits()) + ` | Out-Null +New-NetFirewallRule -DisplayName 'Nylon e2e data network' -Direction Inbound -InterfaceAlias $adapter.Name -Action Allow -Profile Any | Out-Null +New-NetFirewallRule -DisplayName 'Nylon e2e data outbound' -Direction Outbound -InterfaceAlias $adapter.Name -Action Allow -Profile Any | Out-Null +[void][IO.Directory]::CreateDirectory(` + psQuote(w.info.WorkDir) + `)` + if _, err := w.runPS(bootCtx, script, ""); err != nil { + return nil, fmt.Errorf("configure Windows test network: %w", err) + } + return w, nil +} + +func qemuValue(s string) string { return strings.ReplaceAll(s, ",", ",,") } + +func (w *QEMUWindows) Info() Info { return w.info } + +// A local privileged Docker helper configures the host TAP without requiring +// sudo in the Go test runner. The existing debug image already has iproute2. +func (w *QEMUWindows) hostNetwork(ctx context.Context, script string, args ...string) error { + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: w.cfg.HostHelperImage, + Entrypoint: []string{"sh", "-c", script, "machine"}, + Cmd: args, + HostConfigModifier: func(h *container.HostConfig) { h.NetworkMode = "host"; h.Privileged = true }, + WaitingFor: wait.ForExit().WithExitTimeout(30 * time.Second), + }, Started: true, + }) + if c != nil { + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = c.Terminate(cleanupCtx) + }() + } + if err != nil { + return fmt.Errorf("host network helper: %w", err) + } + state, err := c.State(ctx) + if err != nil { + return err + } + if state.ExitCode != 0 { + logs, logErr := c.Logs(ctx) + var output []byte + if logErr == nil { + defer logs.Close() + output, _ = io.ReadAll(logs) + } + return fmt.Errorf("host network helper exited %d: %s", state.ExitCode, output) + } + return nil +} + +func (w *QEMUWindows) Close(ctx context.Context) error { + var closeErr error + if w.done != nil { + select { + case <-w.done: + default: + // Stop QEMU before removing its writable disk. A canceled cleanup can be retried. + if err := w.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + closeErr = errors.Join(closeErr, err) + } + select { + case <-w.done: + case <-ctx.Done(): + return errors.Join(closeErr, ctx.Err()) + } + } + } + if w.log != nil { + closeErr = errors.Join(closeErr, w.log.Close()) + w.log = nil + } + if w.tap != "" { + err := w.hostNetwork(ctx, `if ip link show dev "$1" >/dev/null 2>&1; then ip link delete dev "$1"; fi`, w.tap) + closeErr = errors.Join(closeErr, err) + if err == nil { + w.tap = "" + } + } + // Keep diagnostics, removing only the disposable disk and firmware vars. + for _, name := range []string{"disk.qcow2", "uefi-vars.fd"} { + if err := os.Remove(filepath.Join(w.dir, name)); err != nil && !os.IsNotExist(err) { + closeErr = errors.Join(closeErr, err) + } + } + return closeErr +} diff --git a/e2e/machine/qemu_other.go b/e2e/machine/qemu_other.go new file mode 100644 index 00000000..9d9804d8 --- /dev/null +++ b/e2e/machine/qemu_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package machine + +import ( + "context" + "fmt" +) + +func NewWindows(context.Context, WindowsConfig) (Machine, error) { + return nil, fmt.Errorf("the QEMU Windows backend requires a local Linux host") +} + +func PrepareWindowsImage(context.Context, WindowsConfig) (string, error) { + return "", fmt.Errorf("the QEMU Windows backend requires a local Linux host") +} diff --git a/e2e/machine/winrm.go b/e2e/machine/winrm.go new file mode 100644 index 00000000..4a26068f --- /dev/null +++ b/e2e/machine/winrm.go @@ -0,0 +1,262 @@ +//go:build linux + +package machine + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/masterzen/winrm" + "github.com/masterzen/winrm/soap" +) + +// Each SOAP request owns its NTLM connection. WinRM sends and long-poll +// receives can run concurrently without sharing an encrypted session or pool. +// Closing the socket also interrupts library calls that do not forward ctx. +type winRMTransport struct { + ctx context.Context + endpoint *winrm.Endpoint +} + +func (t *winRMTransport) Transport(endpoint *winrm.Endpoint) error { + t.endpoint = endpoint + return nil +} + +func (t *winRMTransport) Post(client *winrm.Client, message *soap.SoapMessage) (string, error) { + if err := t.ctx.Err(); err != nil { + return "", err + } + address := net.JoinHostPort(t.endpoint.Host, strconv.Itoa(t.endpoint.Port)) + conn, err := (&net.Dialer{Timeout: 20 * time.Second}).DialContext(t.ctx, "tcp", address) + if err != nil { + return "", err + } + defer conn.Close() + stop := context.AfterFunc(t.ctx, func() { _ = conn.Close() }) + defer stop() + transport := winrm.NewClientNTLMWithDial(func(string, string) (net.Conn, error) { return conn, nil }) + if err := transport.Transport(t.endpoint); err != nil { + return "", err + } + response, err := transport.Post(client, message) + if t.ctx.Err() != nil { + return "", t.ctx.Err() + } + return response, err +} + +// runPS uses an encoded command, so scripts never pass through cmd.exe quoting. +func (w *QEMUWindows) runPS(ctx context.Context, script string, input string) (Result, error) { + script = `$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue'; +[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false) +$global:LASTEXITCODE = 0 +try { ` + script + ` +exit $LASTEXITCODE +} catch { [Console]::Error.WriteLine($_.ToString()); exit 1 }` + ctx, cancel := context.WithCancel(ctx) + defer cancel() + params := *winrm.DefaultParameters + params.Timeout = "PT10S" + params.TransportDecorator = func() winrm.Transporter { return &winRMTransport{ctx: ctx} } + endpoint := winrm.NewEndpoint("127.0.0.1", w.port, false, false, nil, nil, nil, 20*time.Second) + client, err := winrm.NewClientWithParameters(endpoint, w.cfg.Username, w.cfg.Password, ¶ms) + if err != nil { + return Result{}, err + } + stdout, stderr, code, err := client.RunPSWithContextWithString(ctx, script, input) + return (Result{Stdout: stdout, Stderr: stderr, ExitCode: code}).check(err) +} + +// windowsArg implements the CommandLineToArgvW/C runtime quoting convention. +func windowsArg(s string) string { + var b strings.Builder + b.WriteByte('"') + slashes := 0 + for _, c := range s { + if c == '\\' { + slashes++ + continue + } + if c == '"' { + b.WriteString(strings.Repeat(`\`, slashes*2+1)) + } else { + b.WriteString(strings.Repeat(`\`, slashes)) + } + slashes = 0 + b.WriteRune(c) + } + b.WriteString(strings.Repeat(`\`, slashes*2)) + b.WriteByte('"') + return b.String() +} + +func windowsArgs(args []string) string { + quoted := make([]string, len(args)) + for i, arg := range args { + quoted[i] = windowsArg(arg) + } + return strings.Join(quoted, " ") +} + +func (w *QEMUWindows) Run(ctx context.Context, cmd Command) (Result, error) { + if cmd.Path == "" { + return Result{}, fmt.Errorf("empty executable") + } + script := `$si = New-Object System.Diagnostics.ProcessStartInfo +$si.UseShellExecute = $false +$si.RedirectStandardOutput = $true +$si.RedirectStandardError = $true +$si.CreateNoWindow = $true +$si.StandardOutputEncoding = New-Object System.Text.UTF8Encoding($false) +$si.StandardErrorEncoding = New-Object System.Text.UTF8Encoding($false) +$si.FileName = ` + psQuote(cmd.Path) + ` +$si.Arguments = ` + psQuote(windowsArgs(cmd.Args)) + "\n" + if cmd.Dir != "" { + script += "$si.WorkingDirectory = " + psQuote(cmd.Dir) + "\n" + } + script += `$p = New-Object System.Diagnostics.Process +$p.StartInfo = $si +try { + [void]$p.Start() + $stdout = $p.StandardOutput.ReadToEndAsync() + $stderr = $p.StandardError.ReadToEndAsync() + $p.WaitForExit() + [Console]::Out.Write($stdout.Result) + [Console]::Error.Write($stderr.Result) + exit $p.ExitCode +} finally { $p.Dispose() }` + return w.runPS(ctx, script, "") +} + +// WriteFile sends base64 lines through one WinRM shell. This avoids a separate +// PowerShell/WinRM session for every small chunk of the Nylon executable. +func (w *QEMUWindows) WriteFile(ctx context.Context, path string, data []byte) error { + expected := fmt.Sprintf("%X", sha256.Sum256(data)) + var input bytes.Buffer + for len(data) != 0 { + n := min(len(data), 36*1024) + input.WriteString(base64.StdEncoding.EncodeToString(data[:n])) + input.WriteByte('\n') + data = data[n:] + } + script := `$path = ` + psQuote(path) + ` +[void][IO.Directory]::CreateDirectory([IO.Path]::GetDirectoryName($path)) +$file = [IO.File]::Open($path, [IO.FileMode]::Create, [IO.FileAccess]::Write) +try { + while ($null -ne ($line = [Console]::ReadLine())) { + $bytes = [Convert]::FromBase64String($line) + $file.Write($bytes, 0, $bytes.Length) + } +} finally { $file.Dispose() } +$sha = [Security.Cryptography.SHA256]::Create() +$stream = [IO.File]::OpenRead($path) +try { [Console]::Out.Write([BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '')) } +finally { $stream.Dispose(); $sha.Dispose() }` + r, err := w.runPS(ctx, script, input.String()) + if err != nil { + return err + } + if strings.TrimSpace(r.Stdout) != expected { + return fmt.Errorf("WinRM upload checksum mismatch for %s", path) + } + return nil +} + +// Shared reads are necessary while the scheduled process is writing its logs. +const psReadShared = `function Read-SharedFile($path) { + $file = [IO.File]::Open($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + $memory = New-Object IO.MemoryStream + try { $file.CopyTo($memory); return [Convert]::ToBase64String($memory.ToArray()) } + finally { $file.Dispose(); $memory.Dispose() } +} +` + +// A scheduled SYSTEM task detaches the process from WinRM's job object. A plain +// Start-Process inside a remote shell would be terminated when that shell closes. +// SYSTEM also gives Nylon the privileges required by its Windows UAPI listener. +func (w *QEMUWindows) Start(ctx context.Context, cmd Command) (Process, error) { + if cmd.Path == "" { + return nil, fmt.Errorf("empty executable") + } + task := "nylon-e2e-" + uuid.NewString() + dir := guestPath(w, task) + stdout, stderr := dir+`\stdout`, dir+`\stderr` + workDir := cmd.Dir + if workDir == "" { + workDir = w.Info().WorkDir + } + argumentList := "" + if len(cmd.Args) != 0 { + argumentList = " -ArgumentList " + psQuote(windowsArgs(cmd.Args)) + } + runner := `$ErrorActionPreference = 'Stop' +try { + $p = Start-Process -FilePath ` + psQuote(cmd.Path) + argumentList + ` -WorkingDirectory ` + psQuote(workDir) + ` -RedirectStandardOutput ` + psQuote(stdout) + ` -RedirectStandardError ` + psQuote(stderr) + ` -PassThru + $handle = $p.Handle + [IO.File]::WriteAllText(` + psQuote(dir+`\pid`) + `, [string]$p.Id) + $p.WaitForExit() + [IO.File]::WriteAllText(` + psQuote(dir+`\exit`) + `, [string]$p.ExitCode) +} catch { + [IO.File]::AppendAllText(` + psQuote(stderr) + `, $_.ToString()) + [IO.File]::WriteAllText(` + psQuote(dir+`\exit`) + `, '127') +}` + if err := w.WriteFile(ctx, dir+`\run.ps1`, []byte(runner)); err != nil { + return nil, err + } + script := `[IO.File]::WriteAllText(` + psQuote(stdout) + `, '') +[IO.File]::WriteAllText(` + psQuote(stderr) + `, '') +$action = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument ` + psQuote(`-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `+windowsArg(dir+`\run.ps1`)) + ` +$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit ([TimeSpan]::Zero) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries +Register-ScheduledTask -TaskName ` + psQuote(task) + ` -Action $action -Principal $principal -Settings $settings | Out-Null +Start-ScheduledTask -TaskName ` + psQuote(task) + ` +for ($i = 0; $i -lt 150; $i++) { + if ((Test-Path -LiteralPath ` + psQuote(dir+`\pid`) + `) -or (Test-Path -LiteralPath ` + psQuote(dir+`\exit`) + `)) { exit 0 } + Start-Sleep -Milliseconds 100 +} +throw 'Scheduled process did not start within 15 seconds'` + if _, err := w.runPS(ctx, script, ""); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + _ = (&windowsProcess{machine: w, dir: dir, task: task}).Stop(cleanupCtx) + return nil, err + } + return &windowsProcess{machine: w, dir: dir, task: task}, nil +} + +type windowsProcess struct { + machine *QEMUWindows + dir, task string +} + +func (p *windowsProcess) Poll(ctx context.Context) (Result, bool, error) { + script := psReadShared + `$code = $null +if (Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `) { $code = [int][IO.File]::ReadAllText(` + psQuote(p.dir+`\exit`) + `) } +@{stdout = (Read-SharedFile ` + psQuote(p.dir+`\stdout`) + `); stderr = (Read-SharedFile ` + psQuote(p.dir+`\stderr`) + `); code = $code} | ConvertTo-Json -Compress` + r, err := p.machine.runPS(ctx, script, "") + if err != nil { + return Result{}, false, err + } + return processOutput(r.Stdout) +} + +func (p *windowsProcess) Stop(ctx context.Context) error { + script := `if (!(Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `) -and (Test-Path -LiteralPath ` + psQuote(p.dir+`\pid`) + `)) { + $child = [int][IO.File]::ReadAllText(` + psQuote(p.dir+`\pid`) + `) + Stop-Process -Id $child -Force -ErrorAction SilentlyContinue +} +Stop-ScheduledTask -TaskName ` + psQuote(p.task) + ` -ErrorAction SilentlyContinue +Unregister-ScheduledTask -TaskName ` + psQuote(p.task) + ` -Confirm:$false -ErrorAction SilentlyContinue +if (!(Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `)) { [IO.File]::WriteAllText(` + psQuote(p.dir+`\exit`) + `, ` + psQuote(strconv.Itoa(-1)) + `) }` + _, err := p.machine.runPS(ctx, script, "") + return err +} diff --git a/e2e/machine_test.go b/e2e/machine_test.go new file mode 100644 index 00000000..937404d4 --- /dev/null +++ b/e2e/machine_test.go @@ -0,0 +1,71 @@ +//go:build e2e + +package e2e + +import ( + "net/netip" + "os" + "testing" + "time" + + "github.com/encodeous/nylon/e2e/machine" + "github.com/encodeous/nylon/protocol" + "github.com/encodeous/nylon/state" +) + +// TestMachines checks Nylon startup, IPC, ping and process shutdown on two peers. +func TestMachines(t *testing.T) { + t.Parallel() + h := NewHarness(t) + dir := h.SetupTestDir() + first, second := state.GenerateKey(), state.GenerateKey() + firstIP, secondIP := GetIP(h.Subnet, 10), GetIP(h.Subnet, 11) + central := state.CentralCfg{ + Routers: []state.RouterCfg{ + SimpleRouter("first", first.Pubkey(), "10.0.0.1", firstIP), + SimpleRouter("second", second.Pubkey(), "10.0.0.2", secondIP), + }, + Graph: []string{"first, second"}, + Timestamp: time.Now().UnixNano(), + } + centralPath := h.WriteConfig(dir, "central.yaml", central) + specs := []NodeSpec{ + {Name: "first", IP: firstIP, CentralConfigPath: centralPath, NodeConfigPath: h.WriteConfig(dir, "first.yaml", SimpleLocal("first", first))}, + {Name: "second", IP: secondIP, CentralConfigPath: centralPath, NodeConfigPath: h.WriteConfig(dir, "second.yaml", SimpleLocal("second", second))}, + } + if os.Getenv("NYLON_E2E_MACHINE") == "mixed" { + specs[0].OS, specs[1].OS = machine.Linux, machine.Windows + } + h.StartMachines(specs...) + for _, peer := range []struct{ name, destination, nextHop string }{ + {"first", "10.0.0.2", "second"}, {"second", "10.0.0.1", "first"}, + } { + h.WaitForStatus(t, peer.name, func(status *protocol.StatusResponse) bool { + return HasSelectedRoute(status, peer.destination+"/32", peer.nextHop, peer.nextHop) + }) + result, err := h.Ping(peer.name, netip.MustParseAddr(peer.destination), machine.PingOptions{Count: 3}) + if err != nil { + h.PrintLogs(peer.name) + t.Fatalf("%s ping: %v\n%s\n%s", peer.name, err, result.Stdout, result.Stderr) + } + } + for name, p := range h.processes { + if err := p.Stop(h.ctx); err != nil { + t.Fatalf("stop %s: %v", name, err) + } + deadline := time.Now().Add(15 * time.Second) + for { + _, exited, err := p.Poll(h.ctx) + if err != nil { + t.Fatalf("poll stopped %s: %v", name, err) + } + if exited { + break + } + if time.Now().After(deadline) { + t.Fatalf("%s did not stop", name) + } + time.Sleep(100 * time.Millisecond) + } + } +} diff --git a/e2e/machines.go b/e2e/machines.go new file mode 100644 index 00000000..655ecd94 --- /dev/null +++ b/e2e/machines.go @@ -0,0 +1,189 @@ +//go:build e2e + +package e2e + +import ( + "fmt" + "net/netip" + "os" + "strings" + "sync" + "time" + + "github.com/encodeous/nylon/e2e/machine" + "github.com/encodeous/nylon/state" + "github.com/goccy/go-yaml" + "github.com/testcontainers/testcontainers-go" +) + +// StartMachines is the portable replacement for StartNodes. Existing fixtures +// may continue using StartNodes/Nodes until their OS-specific operations migrate. +func (h *Harness) StartMachines(specs ...NodeSpec) { + h.t.Helper() + names := make(map[string]bool) + ips := make(map[string]bool) + h.mu.Lock() + for name := range h.Machines { + names[name] = true + } + h.mu.Unlock() + needsWindows := false + for i := range specs { + spec := &specs[i] + if spec.OS == "" { + spec.OS = machine.OS(os.Getenv("NYLON_E2E_MACHINE")) + if spec.OS == "" { + spec.OS = machine.Linux + } + } + if spec.OS != machine.Linux && spec.OS != machine.Windows { + h.t.Fatalf("unsupported machine OS %q (use linux or windows)", spec.OS) + } + needsWindows = needsWindows || spec.OS == machine.Windows + if spec.Name == "" || names[spec.Name] { + h.t.Fatalf("empty or duplicate machine name %q", spec.Name) + } + names[spec.Name] = true + if spec.IP != "" { + if ips[spec.IP] { + h.t.Fatalf("duplicate machine data IP %q", spec.IP) + } + ips[spec.IP] = true + } + } + h.t.Log("Running machine preflight") + prepared, err := h.preflight(needsWindows) + if err != nil { + h.t.Fatalf("machine preflight: %v", err) + } + errs := make(chan error, len(specs)) + var wg sync.WaitGroup + for _, spec := range specs { + wg.Add(1) + go func() { + defer wg.Done() + if err := h.startMachine(spec, prepared); err != nil { + errs <- fmt.Errorf("machine %s: %w", spec.Name, err) + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + h.t.Error(err) + } + if h.t.Failed() { + h.t.FailNow() + } +} + +func (h *Harness) startMachine(spec NodeSpec, prepared windowsPrerequisites) error { + osType := spec.OS + var m machine.Machine + var opts machine.NylonOptions + opts.ExtraArgs = spec.ExtraArgs + var err error + h.t.Logf("Starting %s machine %s at %s", osType, spec.Name, spec.IP) + switch osType { + case machine.Linux: + m, err = machine.NewDocker(h.ctx, machine.DockerConfig{Name: spec.Name, Image: ImageName, Network: h.Network.Name, IP: spec.IP}) + case machine.Windows: + addr, parseErr := netip.ParseAddr(spec.IP) + if parseErr != nil || !addr.Is4() || addr.As4()[3] < 2 || addr.As4()[3] >= 128 { + return fmt.Errorf("Windows requires an explicit data IP in hosts .2 through .127") + } + cfg := prepared.config + opts.Binary, opts.WintunDLL = prepared.binary, prepared.dll + cfg.Name, cfg.IP = spec.Name, spec.IP + m, err = machine.NewWindows(h.ctx, cfg) + opts.StartupTimeout = time.Minute + } + if err != nil { + return err + } + h.t.Logf("Machine %s is ready; installing Nylon and configuration", spec.Name) + // Register immediately, so config/upload/start failures still get cleanup. + h.mu.Lock() + h.Machines[spec.Name] = m + h.mu.Unlock() + centralBytes, err := os.ReadFile(spec.CentralConfigPath) + if err != nil { + return err + } + localBytes, err := os.ReadFile(spec.NodeConfigPath) + if err != nil { + return err + } + var central state.CentralCfg + var local state.LocalCfg + if err := yaml.Unmarshal(centralBytes, ¢ral); err != nil { + return err + } + if err := yaml.Unmarshal(localBytes, &local); err != nil { + return err + } + p, err := machine.RunNylon(h.ctx, m, central, local, opts) + if err != nil { + return err + } + h.t.Logf("Nylon initialized on %s", spec.Name) + h.mu.Lock() + h.processes[spec.Name] = p + h.mu.Unlock() + h.followProcess(spec.Name, p) + return nil +} + +func (h *Harness) followProcess(name string, p machine.Process) { + h.logWG.Add(1) + go func() { + defer h.logWG.Done() + var offsets [2]int + consumer := UnifiedLogConsumer{Node: name, Manager: h.LogManager} + for { + r, exited, err := p.Poll(h.ctx) + if err != nil { + if h.ctx.Err() == nil { + h.t.Logf("read logs for %s: %v", name, err) + } + return + } + for i, data := range []string{r.Stdout, r.Stderr} { + chunk := data[offsets[i]:] + end := strings.LastIndexByte(chunk, '\n') + 1 + if exited { + end = len(chunk) + } + // Preserve per-line matching, including Windows logs on stderr. + for _, line := range strings.SplitAfter(chunk[:end], "\n") { + if line == "" { + continue + } + consumer.Accept(testcontainers.Log{LogType: "stdout", Content: []byte(line)}) + if i == 1 { + h.LogManager.Accept(name, SourceStderr, StripAnsi(line)) + } + } + offsets[i] += end + } + if exited { + return + } + select { + case <-h.ctx.Done(): + return + case <-time.After(500 * time.Millisecond): + } + } + }() +} + +func (h *Harness) Ping(name string, addr netip.Addr, opts machine.PingOptions) (machine.Result, error) { + h.mu.Lock() + m := h.Machines[name] + h.mu.Unlock() + if m == nil { + return machine.Result{}, fmt.Errorf("machine %q not found", name) + } + return machine.Ping(h.ctx, m, addr, opts) +} diff --git a/e2e/preflight.go b/e2e/preflight.go new file mode 100644 index 00000000..dff91bad --- /dev/null +++ b/e2e/preflight.go @@ -0,0 +1,162 @@ +//go:build e2e + +package e2e + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sync" + "time" + + "github.com/encodeous/nylon/e2e/machine" + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" +) + +type windowsPrerequisites struct { + config machine.WindowsConfig + binary, dll string +} + +var windowsPreflight struct { + sync.Once + prepared windowsPrerequisites + err error +} + +// preflight completes shared preparation before StartMachines launches goroutines. +// Linux scenarios need no Windows artifacts. Explicit Windows NodeSpec overrides +// take this path too, regardless of the default placement. +func (h *Harness) preflight(needsWindows bool) (windowsPrerequisites, error) { + if !needsWindows { + return windowsPrerequisites{}, nil + } + windowsPreflight.Do(func() { + fmt.Println("Preflight: preparing Windows base image and Nylon artifacts...") + cfg, err := machine.WindowsConfigFromEnv(h.RootDir) + if err == nil { + cfg.BaseImage, err = machine.PrepareWindowsImage(h.ctx, cfg) + } + prepared := windowsPrerequisites{config: cfg} + if err == nil { + prepared.binary, prepared.dll, err = prepareWindowsArtifacts(h.ctx, h.RootDir) + } + windowsPreflight.prepared, windowsPreflight.err = prepared, err + }) + if windowsPreflight.err != nil { + return windowsPrerequisites{}, windowsPreflight.err + } + provider, err := testcontainers.NewDockerProvider() + if err != nil { + return windowsPrerequisites{}, err + } + defer provider.Close() + inspection, err := provider.Client().NetworkInspect(h.ctx, h.Network.ID, client.NetworkInspectOptions{}) + if err != nil { + return windowsPrerequisites{}, err + } + prepared := windowsPreflight.prepared + prepared.config.Subnet, prepared.config.HostHelperImage = h.Subnet, ImageName + prepared.config.Bridge = inspection.Network.Options["com.docker.network.bridge.name"] + if prepared.config.Bridge == "" { + prepared.config.Bridge = "br-" + inspection.Network.ID[:12] + } + return prepared, nil +} + +func prepareWindowsArtifacts(ctx context.Context, root string) (string, string, error) { + dir := filepath.Join(root, "workdir", "e2e", "artifacts") + if err := os.MkdirAll(dir, 0700); err != nil { + return "", "", err + } + binary := os.Getenv("NYLON_E2E_WINDOWS_BINARY") + if binary != "" && !filepath.IsAbs(binary) { + binary = filepath.Join(root, binary) + } + if binary == "" { + binary = filepath.Join(dir, "nylon.exe") + cmd := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") + cmd.Dir = root + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=windows", "GOARCH=amd64") + if out, err := cmd.CombinedOutput(); err != nil { + return "", "", fmt.Errorf("build Windows Nylon: %w: %s", err, out) + } + } + if _, err := os.Stat(binary); err != nil { + return "", "", err + } + dll := os.Getenv("NYLON_E2E_WINTUN_DLL") + if dll != "" { + if !filepath.IsAbs(dll) { + dll = filepath.Join(root, dll) + } + if _, err := os.Stat(dll); err != nil { + return "", "", err + } + return binary, dll, nil + } + // Match the version and architecture already used by the release workflow. + dll = filepath.Join(dir, "wintun-0.14.1-amd64.dll") + if _, err := os.Stat(dll); err == nil { + return binary, dll, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.wintun.net/builds/wintun-0.14.1.zip", nil) + if err != nil { + return "", "", err + } + resp, err := (&http.Client{Timeout: 2 * time.Minute}).Do(req) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("download Wintun: HTTP %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return "", "", err + } + archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", "", err + } + for _, entry := range archive.File { + if entry.Name != "wintun/bin/amd64/wintun.dll" { + continue + } + reader, err := entry.Open() + if err != nil { + return "", "", err + } + defer reader.Close() + data, err := io.ReadAll(io.LimitReader(reader, 16<<20)) + if err != nil { + return "", "", err + } + tmp, err := os.CreateTemp(dir, ".wintun-*.dll") + if err != nil { + return "", "", err + } + defer os.Remove(tmp.Name()) + _, writeErr := tmp.Write(data) + closeErr := tmp.Close() + if writeErr != nil { + return "", "", writeErr + } + if closeErr != nil { + return "", "", closeErr + } + if err := os.Rename(tmp.Name(), dll); err != nil { + return "", "", err + } + return binary, dll, nil + } + return "", "", fmt.Errorf("Wintun archive has no amd64 DLL") +} diff --git a/go.mod b/go.mod index 8e9e64b0..27f40c89 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/encodeous/nylon go 1.26.3 require ( + github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/digineo/go-ping v1.2.0 github.com/dustin/go-broadcast v0.0.0-20211018055107-71439988bd91 github.com/encodeous/metric v0.0.0-20251111175231-f339c2f7c4bd @@ -11,8 +12,12 @@ require ( github.com/gaissmai/bart v0.25.0 github.com/goccy/go-yaml v1.19.1 github.com/google/go-cmp v0.7.0 + github.com/google/uuid v1.6.0 github.com/jellydator/ttlcache/v3 v3.4.0 + github.com/joho/godotenv v1.5.1 + github.com/klauspost/compress v1.18.7 github.com/kmahyyg/go-network-compo v0.2.10 + github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59 github.com/moby/moby/api v1.54.2 github.com/moby/moby/client v0.4.1 github.com/moby/term v0.5.2 @@ -32,7 +37,10 @@ require ( dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c // indirect + github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect @@ -50,11 +58,18 @@ require ( github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.18.7 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/goidentity/v6 v6.0.1 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect + github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.3.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect diff --git a/go.sum b/go.sum index ea771222..9732b5de 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,16 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c h1:HyK5+m8TFO4iAmMJ7u8tyUjE9C/y4/mvFRT0nqEPfJY= +github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= +github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 h1:w0E0fgc1YafGEh5cROhlROMWXiNoZqApk2PDN0M1+Ns= +github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -25,6 +33,8 @@ github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/digineo/go-logwrap v0.0.0-20181106161722-a178c58ea3f0 h1:OT/LKmj81wMymnWXaKaKBR9n1vPlu+GC0VVKaZP6kzs= @@ -60,6 +70,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/goccy/go-yaml v1.19.1 h1:3rG3+v8pkhRqoQ/88NYNMHYVGYztCOCIZ7UQhu7H+NE= github.com/goccy/go-yaml v1.19.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -67,10 +79,31 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kmahyyg/go-network-compo v0.2.10 h1:H5rZ59xxmaZsIucz5FGV56ww+nzR3k1RFaKEN2bYRog= @@ -83,6 +116,10 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 h1:2ZKn+w/BJeL43sCxI2jhPLRv73oVVOjEKZjKkflyqxg= +github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= +github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59 h1:ZvQGbOpXorIxtua/7uFpXUZ1FyarecLn9BoxqJI6miY= +github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59/go.mod h1:56p25CXdyKsQWriUDfygyVGv5OvFBDY27/yzRy7NuSs= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.3.0 h1:nos4BtzzUIqB406BgQnWGMI4qib9BZ8XUHU+ucv/n1c= @@ -105,6 +142,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= @@ -127,8 +165,15 @@ github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= @@ -137,6 +182,7 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -159,23 +205,52 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= @@ -184,6 +259,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= From 326e921dcc2120c8846017c8329d88692af5f50d Mon Sep 17 00:00:00 2001 From: Dongruixuan Li Date: Sun, 1 Nov 2026 19:09:09 -0500 Subject: [PATCH 2/4] refactor(e2e): target a fixed Linux CI environment --- e2e/README.md | 102 ++++-------- e2e/connectivity_test.go | 6 +- e2e/harness.go | 11 +- e2e/machine/config.go | 89 ----------- e2e/machine/docker.go | 107 ++----------- e2e/machine/image_linux.go | 138 ---------------- e2e/machine/image_linux_test.go | 149 ------------------ e2e/machine/lifecycle_linux_test.go | 83 ---------- e2e/machine/machine.go | 1 - e2e/machine/operations.go | 85 ++-------- e2e/machine/operations_test.go | 38 ----- e2e/machine/qemu_linux.go | 234 ++++++---------------------- e2e/machine/qemu_other.go | 16 -- e2e/machine/winrm.go | 187 +++++----------------- e2e/machine_test.go | 4 +- e2e/machines.go | 105 ++++--------- e2e/preflight.go | 199 ++++++++++------------- go.mod | 2 +- 18 files changed, 274 insertions(+), 1282 deletions(-) delete mode 100644 e2e/machine/config.go delete mode 100644 e2e/machine/image_linux.go delete mode 100644 e2e/machine/image_linux_test.go delete mode 100644 e2e/machine/lifecycle_linux_test.go delete mode 100644 e2e/machine/operations_test.go delete mode 100644 e2e/machine/qemu_other.go diff --git a/e2e/README.md b/e2e/README.md index 2b9051ee..95965e07 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,82 +1,40 @@ -# End-to-end machines +# Portable e2e machines -The Linux test runner can launch Nylon in Linux Docker containers or Windows -QEMU VMs. Use `StartMachines`, `machine.RunNylon` and `Harness.Ping` for portable -scenarios; existing tests and auxiliary services can keep using `StartNodes`. -`NodeSpec.OS` overrides `NYLON_E2E_MACHINE` for an individual node. +`StartMachines`, `machine.RunNylon` and `Harness.Ping` run the same scenarios on +Linux Docker containers and QEMU Windows guests. `NodeSpec.OS` selects individual +nodes; `NYLON_E2E_MACHINE` selects the default. Auxiliary services and unmigrated +Linux tests keep using `StartNodes`. ```sh -# Existing Linux suite -mise run e2e - -# Windows startup, IPC, ping, process shutdown and three-node forwarding NYLON_E2E_MACHINE=windows go test -tags=e2e ./e2e \ -run '^(TestMachines|TestConnectivity)$' -count=1 -parallel=1 -timeout=20m -v - -# Linux endpoints and a Windows relay NYLON_E2E_MACHINE=mixed go test -tags=e2e ./e2e \ -run '^TestConnectivity$' -count=1 -parallel=1 -timeout=20m -v - -# Unit tests, without Docker or booting Windows -go test -race ./e2e/machine ``` -## Setup - -Windows requires a local rootful Linux Docker daemon, `qemu-system-x86_64`, -`qemu-img`, `/dev/net/tun`, writable `/dev/kvm` and OVMF. On Ubuntu install -`qemu-system-x86 qemu-utils ovmf`. Privileged Docker helpers manage TAPs; -QEMU runs as the test user. The image must boot with UEFI, have virtio NIC -and DHCP support, and expose WinRM HTTP/NTLM on port 5985. -The supplied image uses `Administrator` / `nylon` as disposable test credentials. - -Preflight downloads and validates the base image, builds Windows amd64 Nylon, -and prepares Wintun 0.14.1 once per test process, before machines start in -parallel. R2 credentials come from the ignored root `.env`; exported variables -win: `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`. -The default object is `nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst`. -Go handles signing and decompression; AWS CLI and zstd tools are not needed. - -The read-only base is cached in `workdir/e2e/images` using the object filename -without `.zst`. Failed downloads are not published. Clear the cache when -replacing the image; simultaneous test processes sharing this cache are not -supported. Each VM gets a fresh qcow2 backed by that image and its own UEFI vars. -Cleanup removes QEMU, TAPs, overlays and vars while preserving the base and -host/guest logs under `workdir/e2e/{vms,logs}`. - -WinRM uploads the binaries and YAML. Each operation creates an independent -client; each SOAP request owns a temporary encrypted NTLM connection. SYSTEM -scheduled tasks keep Nylon alive after the WinRM shell closes and provide UAPI -permissions. Uploads are checked with SHA256. - -## Networking - -Each scenario has a Docker bridge and `/24`. VM TAPs join that bridge; Docker -allocates dynamic addresses from `.128/25`, while explicit node IPs use `.2`–`.127`. -Each VM also has a restricted management NIC with localhost-only WinRM forwarding; -it cannot carry test traffic through management NAT. The data NIC has no default -gateway. The firewall stays enabled, with allow rules on data and Nylon interfaces. -Docker DNS aliases do not extend to VMs: use service IPs or an explicit resolver. - -Windows keeps selected route prefixes separate; Linux/macOS still merge them. -This avoids adjacent host addresses becoming an on-link subnet whose upper IP -Windows treats as broadcast. Explicitly advertised subnets remain subnet routes. - -## Overrides - -| Variable | Default / purpose | -| --- | --- | -| `NYLON_E2E_WINDOWS_IMAGE` | Existing standalone qcow2; bypass R2 | -| `NYLON_E2E_IMAGE_CACHE` | `workdir/e2e/images` | -| `NYLON_E2E_WINDOWS_BUCKET` / `NYLON_E2E_WINDOWS_OBJECT` | R2 bucket/object above | -| `NYLON_E2E_WINRM_USERNAME` / `NYLON_E2E_WINRM_PASSWORD` | `Administrator` / `nylon` | -| `NYLON_E2E_WINDOWS_MEMORY_MIB` / `NYLON_E2E_WINDOWS_CPUS` | `2048` / `2` per VM | -| `NYLON_E2E_QEMU_ACCEL` / `NYLON_E2E_QEMU_NIC` | `kvm` / `virtio-net-pci`; `tcg` is opt-in | -| `NYLON_E2E_UEFI_CODE` / `NYLON_E2E_UEFI_VARS` | `/usr/share/OVMF/OVMF_{CODE,VARS}_4M.fd` | -| `NYLON_E2E_WINDOWS_BINARY` / `NYLON_E2E_WINTUN_DLL` | Existing Windows amd64 binaries | - -## CI - -CI integration will be added separately once Vault credentials are configured. -The local commands above can be used on a Linux runner with Docker and KVM. +The runner is Linux with a local rootful Docker daemon/CLI, writable `/dev/kvm`, +`/dev/net/tun`, `qemu-system-x86_64`, `qemu-img`, and Ubuntu's OVMF 4M firmware. +On Ubuntu install `qemu-system-x86 qemu-utils ovmf`. Guests use fixed settings: +Windows Server 2025, UEFI, virtio NICs, 2 CPUs, 2 GiB RAM and WinRM HTTP/NTLM on +port 5985 as `Administrator` / `nylon`. The image includes the NIC drivers. + +Preflight downloads `nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst` from R2, +builds Windows amd64 Nylon, and prepares Wintun 0.14.1 once before parallel boot. +R2 credentials are `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, and +`AWS_SECRET_ACCESS_KEY`, supplied as environment variables or in the ignored root +`.env`. The image is cached in `workdir/e2e/images`; clear it to replace the image. +Each VM uses a new qcow2 overlay and UEFI variables; cleanup removes both and +retains diagnostics under `workdir/e2e/{vms,logs}`. + +A TAP joins each data NIC to the scenario's `/24` Docker bridge. Docker assigns +addresses from `.128/25`; nodes use explicit addresses in `.2`–`.127`. A separate +restricted management NIC forwards WinRM to localhost and cannot carry test +traffic. Disposable guests have their firewall disabled. Each machine runs one +Nylon process, launched as a SYSTEM task on Windows; uploads use SHA256 checks. + +Windows installs selected prefixes separately to avoid merging adjacent hosts +into on-link subnets with broadcast addresses. Linux/macOS retain aggregation; +explicitly advertised subnets remain subnet routes. + +CI workflow integration will follow separately once Vault credentials are ready. Allow 4 GiB of guest RAM for two Windows nodes or 6 GiB for three. diff --git a/e2e/connectivity_test.go b/e2e/connectivity_test.go index 73296cf6..b34e1f87 100644 --- a/e2e/connectivity_test.go +++ b/e2e/connectivity_test.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e && linux package e2e @@ -86,7 +86,7 @@ func TestConnectivity(t *testing.T) { // Wait for the data plane before asserting consecutive successful replies. deadline := time.Now().Add(WaitTimeout) for { - result, err := h.Ping("node3", netip.MustParseAddr(node1NylonIP), machine.PingOptions{Count: 1}) + result, err := h.Ping("node3", netip.MustParseAddr(node1NylonIP), 1) if err == nil { break } @@ -101,7 +101,7 @@ func TestConnectivity(t *testing.T) { {"node3", node1NylonIP}, {"node2", node1NylonIP}, {"node1", node3NylonIP}, } { t.Logf("Pinging %s from %s...", peer.to, peer.from) - result, err := h.Ping(peer.from, netip.MustParseAddr(peer.to), machine.PingOptions{Count: 3}) + result, err := h.Ping(peer.from, netip.MustParseAddr(peer.to), 3) if err != nil { for _, node := range []string{"node1", "node2", "node3"} { h.PrintLogs(node) diff --git a/e2e/harness.go b/e2e/harness.go index adce98fc..be650841 100644 --- a/e2e/harness.go +++ b/e2e/harness.go @@ -430,15 +430,15 @@ func (h *Harness) Cleanup() { h.logWG.Wait() h.mu.Lock() defer h.mu.Unlock() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() if len(h.processes) != 0 { dir := filepath.Join(h.RootDir, "workdir", "e2e", "logs", h.t.Name()) if err := os.MkdirAll(dir, 0700); err != nil { h.t.Logf("create machine log directory: %v", err) } else { for name, p := range h.processes { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) r, _, err := p.Poll(cleanupCtx) - cancel() if err != nil { h.t.Logf("save machine logs for %s: %v", name, err) continue @@ -452,17 +452,13 @@ func (h *Harness) Cleanup() { } } for name, m := range h.Machines { - cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) err := m.Close(cleanupCtx) - cancel() if err != nil { h.t.Logf("failed to close machine %s: %v", name, err) } } for name, c := range h.Nodes { - cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) err := c.Terminate(cleanupCtx) - cancel() if err != nil { h.t.Logf("failed to terminate container %s: %v", name, err) } @@ -477,9 +473,6 @@ func (h *Harness) Exec(nodeName string, cmd []string) (string, string, error) { container, ok := h.Nodes[nodeName] h.mu.Unlock() if m != nil { - if len(cmd) == 0 { - return "", "", fmt.Errorf("empty command") - } path := cmd[0] if path == "nylon" && m.Info().OS == machine.Windows { path = m.Info().WorkDir + `\nylon.exe` diff --git a/e2e/machine/config.go b/e2e/machine/config.go deleted file mode 100644 index 8b09ab89..00000000 --- a/e2e/machine/config.go +++ /dev/null @@ -1,89 +0,0 @@ -package machine - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/joho/godotenv" -) - -type WindowsConfig struct { - Name, IP, Subnet, Bridge string - // HostHelperImage supplies iproute2 in a privileged, host-networked Docker - // helper. The Docker daemon must run locally on this Linux QEMU host. - HostHelperImage string - WorkDir string - BaseImage string - CacheDir string - R2Endpoint string - R2AccessKey string - R2SecretKey string - Bucket string - Object string - Username string - Password string - MemoryMiB int - CPUs int - Accelerator string - NICModel string - UEFICode string - UEFIVars string - BootTimeout time.Duration -} - -// WindowsConfigFromEnv reads .env without changing the process environment. -// Exported environment variables take precedence, so the same code works in CI. -func WindowsConfigFromEnv(root string) (WindowsConfig, error) { - values, err := godotenv.Read(filepath.Join(root, ".env")) - if err != nil && !os.IsNotExist(err) { - return WindowsConfig{}, err - } - get := func(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - if value, ok := values[key]; ok { - return value - } - return fallback - } - cfg := WindowsConfig{ - BaseImage: get("NYLON_E2E_WINDOWS_IMAGE", ""), - CacheDir: get("NYLON_E2E_IMAGE_CACHE", filepath.Join(root, "workdir", "e2e", "images")), - WorkDir: filepath.Join(root, "workdir", "e2e", "vms"), - R2Endpoint: get("AWS_ENDPOINT_URL_S3", ""), - R2AccessKey: get("AWS_ACCESS_KEY_ID", ""), - R2SecretKey: get("AWS_SECRET_ACCESS_KEY", ""), - Bucket: get("NYLON_E2E_WINDOWS_BUCKET", "nylon-ci-windows-image"), - Object: get("NYLON_E2E_WINDOWS_OBJECT", "nylon-ci-win2025.qcow2.zst"), - Username: get("NYLON_E2E_WINRM_USERNAME", "Administrator"), - Password: get("NYLON_E2E_WINRM_PASSWORD", "nylon"), - Accelerator: get("NYLON_E2E_QEMU_ACCEL", "kvm"), - NICModel: get("NYLON_E2E_QEMU_NIC", "virtio-net-pci"), - UEFICode: get("NYLON_E2E_UEFI_CODE", "/usr/share/OVMF/OVMF_CODE_4M.fd"), - UEFIVars: get("NYLON_E2E_UEFI_VARS", "/usr/share/OVMF/OVMF_VARS_4M.fd"), - BootTimeout: 5 * time.Minute, - } - for _, field := range []struct { - key, fallback string - dst *int - }{ - {"NYLON_E2E_WINDOWS_MEMORY_MIB", "2048", &cfg.MemoryMiB}, - {"NYLON_E2E_WINDOWS_CPUS", "2", &cfg.CPUs}, - } { - value, err := strconv.Atoi(get(field.key, field.fallback)) - if err != nil || value < 1 { - return WindowsConfig{}, fmt.Errorf("%s must be a positive integer", field.key) - } - *field.dst = value - } - for _, path := range []*string{&cfg.BaseImage, &cfg.CacheDir, &cfg.UEFICode, &cfg.UEFIVars} { - if *path != "" && !filepath.IsAbs(*path) { - *path = filepath.Join(root, *path) - } - } - return cfg, nil -} diff --git a/e2e/machine/docker.go b/e2e/machine/docker.go index 37288530..eb8563d7 100644 --- a/e2e/machine/docker.go +++ b/e2e/machine/docker.go @@ -3,22 +3,17 @@ package machine import ( "bytes" "context" - "fmt" "net/netip" + "os/exec" "strings" - "time" - "github.com/google/uuid" - "github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/network" - "github.com/moby/moby/client" "github.com/testcontainers/testcontainers-go" ) type DockerConfig struct { Name string - Image string Network string IP string } @@ -26,21 +21,12 @@ type DockerConfig struct { type Docker struct { info Info Container testcontainers.Container - provider *testcontainers.DockerProvider } func NewDocker(ctx context.Context, cfg DockerConfig) (*Docker, error) { - if cfg.Image == "" || cfg.Network == "" { - return nil, fmt.Errorf("Docker machine requires an image and network") - } - if cfg.IP != "" { - if _, err := netip.ParseAddr(cfg.IP); err != nil { - return nil, err - } - } c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Image: cfg.Image, + Image: "nylon-debug:latest", Entrypoint: []string{"/bin/sh", "-c", "exec sleep infinity"}, Networks: []string{cfg.Network}, NetworkAliases: map[string][]string{cfg.Network: {cfg.Name}}, @@ -49,75 +35,30 @@ func NewDocker(ctx context.Context, cfg DockerConfig) (*Docker, error) { h.CapAdd = []string{"NET_ADMIN"} }, EndpointSettingsModifier: func(endpoints map[string]*network.EndpointSettings) { - if cfg.IP != "" && endpoints[cfg.Network] != nil { - endpoints[cfg.Network].IPAMConfig = &network.EndpointIPAMConfig{IPv4Address: netip.MustParseAddr(cfg.IP)} - } + endpoints[cfg.Network].IPAMConfig = &network.EndpointIPAMConfig{IPv4Address: netip.MustParseAddr(cfg.IP)} }, }, Started: true, }) if err != nil { - if c != nil { - _ = c.Terminate(context.Background()) - } return nil, err } - if cfg.IP == "" { - cfg.IP, err = c.ContainerIP(ctx) - if err != nil { - _ = c.Terminate(context.Background()) - return nil, err - } - } - provider, err := testcontainers.NewDockerProvider() - if err != nil { - _ = c.Terminate(context.Background()) - return nil, err - } - return &Docker{info: Info{OS: Linux, IP: cfg.IP, WorkDir: "/app/config"}, Container: c, provider: provider}, nil + return &Docker{info: Info{OS: Linux, IP: cfg.IP, WorkDir: "/app/config"}, Container: c}, nil } func (d *Docker) Info() Info { return d.info } -func (d *Docker) Run(ctx context.Context, cmd Command) (Result, error) { - if cmd.Path == "" { - return Result{}, fmt.Errorf("empty executable") - } - cli := d.provider.Client() - created, err := cli.ExecCreate(ctx, d.Container.GetContainerID(), client.ExecCreateOptions{ - Cmd: append([]string{cmd.Path}, cmd.Args...), WorkingDir: cmd.Dir, AttachStdout: true, AttachStderr: true, - }) - if err != nil { - return Result{}, err - } - attached, err := cli.ExecAttach(ctx, created.ID, client.ExecAttachOptions{}) - if err != nil { - return Result{}, err - } - defer attached.Close() - stop := context.AfterFunc(ctx, attached.Close) - defer stop() +func (d *Docker) Run(ctx context.Context, command Command) (Result, error) { + args := append([]string{"exec", d.Container.GetContainerID(), command.Path}, command.Args...) + cmd := exec.CommandContext(ctx, "docker", args...) var stdout, stderr bytes.Buffer - // Drain while the process runs, so output larger than Docker's socket buffer - // cannot deadlock the process before an exit-status inspection. - _, err = stdcopy.StdCopy(&stdout, &stderr, attached.Reader) - r := Result{Stdout: stdout.String(), Stderr: stderr.String()} - if err != nil { - return r, err - } - for { - status, err := cli.ExecInspect(ctx, created.ID, client.ExecInspectOptions{}) - if err != nil { - return r, err - } - if !status.Running { - r.ExitCode = status.ExitCode - return r.check(nil) - } - if err := pause(ctx, 10*time.Millisecond); err != nil { - return r, err - } + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + result := Result{Stdout: stdout.String(), Stderr: stderr.String()} + if cmd.ProcessState != nil { + result.ExitCode = cmd.ProcessState.ExitCode() } + return result.check(err) } func (d *Docker) WriteFile(ctx context.Context, path string, data []byte) error { @@ -128,21 +69,12 @@ func (d *Docker) WriteFile(ctx context.Context, path string, data []byte) error } func (d *Docker) Start(ctx context.Context, cmd Command) (Process, error) { - if cmd.Path == "" { - return nil, fmt.Errorf("empty executable") - } - dir := "/tmp/nylon-process-" + uuid.NewString() - if _, err := d.Run(ctx, Command{Path: "mkdir", Args: []string{"-p", dir}}); err != nil { - return nil, err - } + dir := d.info.WorkDir args := []string{shellQuote(cmd.Path)} for _, a := range cmd.Args { args = append(args, shellQuote(a)) } - script := "#!/bin/sh\n" - if cmd.Dir != "" { - script += "cd -- " + shellQuote(cmd.Dir) + " || exit 127\n" - } + script := "#!/bin/sh\ncd " + shellQuote(dir) + "\n" script += strings.Join(args, " ") + " >" + shellQuote(dir+"/stdout") + " 2>" + shellQuote(dir+"/stderr") + " &\n" script += "child=$!\nprintf '%s' \"$child\" >" + shellQuote(dir+"/pid") + "\nwait \"$child\"\ncode=$?\nprintf '%s' \"$code\" >" + shellQuote(dir+"/exit") + "\n" if err := d.WriteFile(ctx, dir+"/run.sh", []byte(script)); err != nil { @@ -150,11 +82,7 @@ func (d *Docker) Start(ctx context.Context, cmd Command) (Process, error) { } _, err := d.Run(ctx, Command{Path: "sh", Args: []string{"-c", `touch "$1/stdout" "$1/stderr" sh "$1/run.sh" /dev/null 2>&1 & -n=0 -while [ ! -s "$1/pid" ] && [ ! -s "$1/exit" ]; do - n=$((n+1)); [ "$n" -lt 100 ] || exit 1 - sleep 0.05 -done`, "machine", dir}}) +`, "machine", dir}}) if err != nil { return nil, err } @@ -162,7 +90,6 @@ done`, "machine", dir}}) } func (d *Docker) Close(ctx context.Context) error { - defer d.provider.Close() return d.Container.Terminate(ctx) } @@ -181,6 +108,6 @@ printf '{"stdout":"%s","stderr":"%s","code":%s}' "$(base64 -w0 "$1/stdout")" "$( } func (p *dockerProcess) Stop(ctx context.Context) error { - _, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `if [ ! -f "$1/exit" ] && [ -f "$1/pid" ]; then kill "$(cat "$1/pid")" 2>/dev/null || true; fi`, "machine", p.dir}}) + _, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `kill "$(cat "$1/pid")"`, "machine", p.dir}}) return err } diff --git a/e2e/machine/image_linux.go b/e2e/machine/image_linux.go deleted file mode 100644 index 19629482..00000000 --- a/e2e/machine/image_linux.go +++ /dev/null @@ -1,138 +0,0 @@ -package machine - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/klauspost/compress/zstd" -) - -// PrepareWindowsImage runs during preflight, before any VMs are started. -// A completed image is reused until the cache is explicitly cleared. -func PrepareWindowsImage(ctx context.Context, cfg WindowsConfig) (string, error) { - if cfg.BaseImage != "" { - path, err := filepath.Abs(cfg.BaseImage) - if err != nil { - return "", err - } - return path, validateBaseImage(ctx, path) - } - cacheDir, err := filepath.Abs(cfg.CacheDir) - if err != nil { - return "", err - } - path := filepath.Join(cacheDir, strings.TrimSuffix(filepath.Base(cfg.Object), ".zst")) - if _, err := os.Stat(path); err == nil { - return path, validateBaseImage(ctx, path) - } - if cfg.R2Endpoint == "" || cfg.R2AccessKey == "" || cfg.R2SecretKey == "" { - return "", fmt.Errorf("set AWS_ENDPOINT_URL_S3, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or NYLON_E2E_WINDOWS_IMAGE") - } - u, err := url.Parse(cfg.R2Endpoint) - if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") { - return "", fmt.Errorf("invalid R2 endpoint") - } - if err := os.MkdirAll(cacheDir, 0700); err != nil { - return "", err - } - objectURL := u.JoinPath(cfg.Bucket, cfg.Object).String() - client := &http.Client{Timeout: 30 * time.Minute} - response, err := r2Request(ctx, client, cfg, objectURL) - if err != nil { - return "", err - } - defer response.Body.Close() - decoder, err := zstd.NewReader(response.Body) - if err != nil { - return "", err - } - defer decoder.Close() - tmp, err := os.CreateTemp(cacheDir, ".windows-*.qcow2.partial") - if err != nil { - return "", err - } - defer os.Remove(tmp.Name()) - defer tmp.Close() - if _, err := io.Copy(tmp, decoder); err != nil { - return "", fmt.Errorf("decompress Windows image: %w", err) - } - if err := tmp.Close(); err != nil { - return "", err - } - if err := validateBaseImage(ctx, tmp.Name()); err != nil { - return "", err - } - if err := os.Chmod(tmp.Name(), 0444); err != nil { - return "", err - } - if err := os.Rename(tmp.Name(), path); err != nil { - return "", err - } - return path, nil -} - -func r2Request(ctx context.Context, client *http.Client, cfg WindowsConfig, objectURL string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, objectURL, nil) - if err != nil { - return nil, err - } - emptyHash := sha256.Sum256(nil) - req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(emptyHash[:])) - err = v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: cfg.R2AccessKey, SecretAccessKey: cfg.R2SecretKey}, req, hex.EncodeToString(emptyHash[:]), "s3", "auto", time.Now()) - if err != nil { - return nil, fmt.Errorf("sign R2 request: %w", err) - } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("request Windows image: %w", err) - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("R2 image: HTTP %d", resp.StatusCode) - } - return resp, nil -} - -func validateBaseImage(ctx context.Context, path string) error { - out, err := exec.CommandContext(ctx, "qemu-img", "info", "--output=json", path).CombinedOutput() - if err != nil { - return fmt.Errorf("inspect base image: %w: %s", err, out) - } - var info struct { - Format string `json:"format"` - Backing string `json:"backing-filename"` - } - if err := json.Unmarshal(out, &info); err != nil { - return err - } - if info.Format != "qcow2" || info.Backing != "" { - return fmt.Errorf("base image must be a standalone qcow2") - } - return nil -} - -// CreateOverlay never writes to the shared backing image. -func CreateOverlay(ctx context.Context, base, overlay string) error { - absolute, err := filepath.Abs(base) - if err != nil { - return err - } - out, err := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", absolute, overlay).CombinedOutput() - if err != nil { - return fmt.Errorf("create qcow2 overlay: %w: %s", err, out) - } - return nil -} diff --git a/e2e/machine/image_linux_test.go b/e2e/machine/image_linux_test.go deleted file mode 100644 index fd8785d0..00000000 --- a/e2e/machine/image_linux_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package machine - -import ( - "crypto/sha256" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strings" - "sync/atomic" - "testing" - - "github.com/klauspost/compress/zstd" -) - -func smallBase(t *testing.T) string { - t.Helper() - if _, err := exec.LookPath("qemu-img"); err != nil { - t.Skip("qemu-img is not installed") - } - path := filepath.Join(t.TempDir(), "base.qcow2") - if out, err := exec.Command("qemu-img", "create", "-f", "qcow2", path, "8M").CombinedOutput(); err != nil { - t.Fatalf("create base: %v: %s", err, out) - } - return path -} - -func TestOverlaysIsolateWritesAndPreserveBase(t *testing.T) { - base := smallBase(t) - if _, err := exec.LookPath("qemu-io"); err != nil { - t.Skip("qemu-io is not installed") - } - before, err := os.ReadFile(base) - if err != nil { - t.Fatal(err) - } - baseHash := sha256.Sum256(before) - paths := []string{filepath.Join(t.TempDir(), "first.qcow2"), filepath.Join(t.TempDir(), "second.qcow2")} - for _, path := range paths { - if err := CreateOverlay(t.Context(), base, path); err != nil { - t.Fatal(err) - } - out, err := exec.Command("qemu-img", "info", "--output=json", path).CombinedOutput() - if err != nil { - t.Fatal(err) - } - var info struct { - Backing string `json:"full-backing-filename"` - Format string `json:"backing-filename-format"` - } - if err := json.Unmarshal(out, &info); err != nil { - t.Fatal(err) - } - if info.Backing != base || info.Format != "qcow2" { - t.Fatalf("unexpected backing: %+v", info) - } - } - if out, err := exec.Command("qemu-io", "-f", "qcow2", "-c", "write -P 0x42 0 512", paths[0]).CombinedOutput(); err != nil { - t.Fatalf("write overlay: %v: %s", err, out) - } - if out, err := exec.Command("qemu-io", "-r", "-f", "qcow2", "-c", "read -P 0 0 512", paths[1]).CombinedOutput(); err != nil { - t.Fatalf("second overlay changed: %v: %s", err, out) - } - after, err := os.ReadFile(base) - if err != nil { - t.Fatal(err) - } - if sha256.Sum256(after) != baseHash { - t.Fatal("backing image was modified") - } -} - -func TestR2CacheRejectsIncompleteDownloadAndReusesImage(t *testing.T) { - base := smallBase(t) - data, err := os.ReadFile(base) - if err != nil { - t.Fatal(err) - } - encoder, err := zstd.NewWriter(nil) - if err != nil { - t.Fatal(err) - } - defer encoder.Close() - compressed := encoder.EncodeAll(data, nil) - var gets atomic.Int32 - var corrupt atomic.Bool - corrupt.Store(true) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 ") || r.Header.Get("X-Amz-Content-Sha256") == "" { - t.Error("unsigned R2 request") - w.WriteHeader(403) - return - } - if r.URL.Path != "/bucket/image.qcow2.zst" { - t.Errorf("unexpected object: %s", r.URL.Path) - } - if r.Method != http.MethodGet { - t.Errorf("unexpected method: %s", r.Method) - } - gets.Add(1) - if corrupt.Load() { - _, _ = w.Write([]byte("interrupted image")) - return - } - _, _ = w.Write(compressed) - })) - defer server.Close() - cfg := WindowsConfig{R2Endpoint: server.URL, R2AccessKey: "test-key", R2SecretKey: "test-secret", Bucket: "bucket", Object: "image.qcow2.zst", CacheDir: t.TempDir()} - if _, err := PrepareWindowsImage(t.Context(), cfg); err == nil { - t.Fatal("corrupt compressed image accepted") - } - files, err := filepath.Glob(filepath.Join(cfg.CacheDir, "*.qcow2")) - if err != nil { - t.Fatal(err) - } - if len(files) != 0 { - t.Fatal("incomplete image published to cache") - } - corrupt.Store(false) - path, err := PrepareWindowsImage(t.Context(), cfg) - if err != nil { - t.Fatal(err) - } - // Warm-cache preflight needs no R2 credentials or network request. - cfg.R2Endpoint, cfg.R2AccessKey, cfg.R2SecretKey = "", "", "" - cached, err := PrepareWindowsImage(t.Context(), cfg) - if err != nil || cached != path { - t.Fatalf("cache reuse: %s, %v", cached, err) - } - if gets.Load() != 2 { - t.Fatalf("got %d downloads; expected one failed and one successful", gets.Load()) - } - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if info.Mode().Perm()&0222 != 0 { - t.Fatal("cached backing image is writable") - } - got, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if sha256.Sum256(got) != sha256.Sum256(data) { - t.Fatal("cached image content changed") - } -} diff --git a/e2e/machine/lifecycle_linux_test.go b/e2e/machine/lifecycle_linux_test.go deleted file mode 100644 index 9c79b641..00000000 --- a/e2e/machine/lifecycle_linux_test.go +++ /dev/null @@ -1,83 +0,0 @@ -//go:build linux - -package machine - -import ( - "context" - "errors" - "net" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strconv" - "testing" - "time" -) - -func TestWinRMHonorsCanceledContext(t *testing.T) { - started := make(chan struct{}) - release := make(chan struct{}) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - w.(http.Flusher).Flush() - close(started) - <-release - })) - defer server.Close() - _, portText, _ := net.SplitHostPort(server.Listener.Addr().String()) - port, _ := strconv.Atoi(portText) - w := QEMUWindows{port: port, cfg: WindowsConfig{Username: "test", Password: "test"}} - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { _, err := w.runPS(ctx, "'hello'", ""); done <- err }() - <-started - cancel() - select { - case err := <-done: - close(release) - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected canceled operation, got %v", err) - } - case <-time.After(time.Second): - t.Errorf("runPS still blocked after context expired: ctx.Err()=%v", ctx.Err()) - close(release) - t.Logf("after HTTP response body was released: %v", <-done) - } -} - -func TestCloseCanRetryAfterCancellation(t *testing.T) { - dir := t.TempDir() - disk := filepath.Join(dir, "disk.qcow2") - vars := filepath.Join(dir, "uefi-vars.fd") - for _, p := range []string{disk, vars} { - if err := os.WriteFile(p, []byte("test"), 0600); err != nil { - t.Fatal(err) - } - } - cmd := exec.Command("sleep", "30") - if err := cmd.Start(); err != nil { - t.Fatal(err) - } - reaped := make(chan struct{}) - done := make(chan struct{}) - w := QEMUWindows{dir: dir, cmd: cmd, done: done} - // Model an already cancelled harness budget and a reaper not scheduled yet. - go func() { _ = cmd.Wait(); close(reaped); <-done }() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - first := w.Close(ctx) - <-reaped - close(done) - second := w.Close(context.Background()) - if second != nil { - t.Fatalf("cleanup retry: %v (first: %v)", second, first) - } - for _, path := range []string{disk, vars} { - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Errorf("cleanup did not remove %s: %v", path, err) - } - } -} diff --git a/e2e/machine/machine.go b/e2e/machine/machine.go index 63dbda39..7cc2fff1 100644 --- a/e2e/machine/machine.go +++ b/e2e/machine/machine.go @@ -28,7 +28,6 @@ type Info struct { type Command struct { Path string Args []string - Dir string } type Result struct { diff --git a/e2e/machine/operations.go b/e2e/machine/operations.go index d91dd8d2..953f26ff 100644 --- a/e2e/machine/operations.go +++ b/e2e/machine/operations.go @@ -3,7 +3,6 @@ package machine import ( "context" "fmt" - "math" "net/netip" "os" "strconv" @@ -16,10 +15,9 @@ import ( type NylonOptions struct { // Host paths prepared by preflight for Windows; Linux uses its Docker image. - Binary string - WintunDLL string - ExtraArgs []string - StartupTimeout time.Duration + Binary string + WintunDLL string + ExtraArgs []string } // RunNylon installs config and optional artifacts, launches Nylon, and waits for @@ -45,9 +43,6 @@ func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local st binary := "/usr/local/bin/nylon" if m.Info().OS == Windows { binary = guestPath(m, "nylon.exe") - if opts.Binary == "" || opts.WintunDLL == "" { - return nil, fmt.Errorf("Windows requires NylonOptions.Binary and WintunDLL") - } for name, source := range map[string]string{"nylon.exe": opts.Binary, "wintun.dll": opts.WintunDLL} { if err := upload(ctx, m, source, guestPath(m, name)); err != nil { return nil, err @@ -56,7 +51,7 @@ func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local st } args := []string{"run", "-v", "-w", "--dbg-trace-tc", "--config", guestPath(m, "central.yaml"), "--node", guestPath(m, "node.yaml")} args = append(args, opts.ExtraArgs...) - p, err := m.Start(ctx, Command{Path: binary, Args: args, Dir: m.Info().WorkDir}) + p, err := m.Start(ctx, Command{Path: binary, Args: args}) if err != nil { return nil, err } @@ -67,16 +62,10 @@ func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local st _ = p.Stop(cleanupCtx) } }() - timeout := opts.StartupTimeout - if timeout == 0 { - timeout = 30 * time.Second - } - waitCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() var output Result for { var exited bool - output, exited, err = p.Poll(waitCtx) + output, exited, err = p.Poll(ctx) if err != nil { return nil, fmt.Errorf("read Nylon startup output: %w", err) } @@ -84,22 +73,9 @@ func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local st return nil, fmt.Errorf("Nylon exited during startup (%d): %s\n%s", output.ExitCode, output.Stdout, output.Stderr) } if strings.Contains(output.Stdout+output.Stderr, "Nylon has been initialized") { - if m.Info().OS == Windows && !local.NoTun { - name := local.InterfaceName - if name == "" { - name = "nylon" - } - // Wintun appears after the machine's data-NIC firewall setup. - // Allow the scenario's inner packets on this disposable interface. - script := "New-NetFirewallRule -DisplayName 'Nylon e2e overlay inbound' -Direction Inbound -InterfaceAlias " + psQuote(name) + " -Action Allow -Profile Any | Out-Null\n" + - "New-NetFirewallRule -DisplayName 'Nylon e2e overlay outbound' -Direction Outbound -InterfaceAlias " + psQuote(name) + " -Action Allow -Profile Any | Out-Null" - if _, err := m.Run(waitCtx, Command{Path: "powershell.exe", Args: []string{"-NoProfile", "-NonInteractive", "-Command", script}}); err != nil { - return nil, fmt.Errorf("configure Windows overlay firewall: %w", err) - } - } return p, nil } - if err := pause(waitCtx, 250*time.Millisecond); err != nil { + if err := pause(ctx, 250*time.Millisecond); err != nil { return nil, fmt.Errorf("waiting for Nylon initialization: %w\n%s\n%s", err, output.Stdout, output.Stderr) } } @@ -116,49 +92,20 @@ func upload(ctx context.Context, m Machine, src, dst string) error { return nil } -type PingOptions struct { - Count int - Timeout time.Duration // Per reply. -} - -// Ping executes the guest's ping against an IP, exercising its real system -// routing and TUN. Failure is determined by replies, including on Windows where -// ping.exe may otherwise return success for a destination-unreachable reply. -func Ping(ctx context.Context, m Machine, addr netip.Addr, opts PingOptions) (Result, error) { - if !addr.IsValid() { - return Result{}, fmt.Errorf("invalid ping address") - } - if opts.Count == 0 { - opts.Count = 3 - } - if opts.Timeout == 0 { - opts.Timeout = 2 * time.Second - } - if opts.Count < 1 || opts.Timeout < 0 { - return Result{}, fmt.Errorf("invalid ping options") - } - // Include intervals and command startup, while respecting an earlier caller deadline. - ctx, cancel := context.WithTimeout(ctx, time.Duration(opts.Count)*(opts.Timeout+time.Second)+15*time.Second) +// Ping requires all requested replies on both platforms. +func Ping(ctx context.Context, m Machine, addr netip.Addr, count int) (Result, error) { + ctx, cancel := context.WithTimeout(ctx, time.Duration(count*3+15)*time.Second) defer cancel() if m.Info().OS == Windows { - // .NET Ping is independent of the language of the Windows installation and - // gives an explicit IPStatus for each reply. It uses the guest network stack. script := fmt.Sprintf(`$p = New-Object System.Net.NetworkInformation.Ping $failed = $false -try { - for ($i = 0; $i -lt %d; $i++) { - $r = $p.Send(%s, %d) - Write-Output ($r.Status.ToString() + ' ' + $r.RoundtripTime + 'ms') - if ($r.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) { $failed = $true } - } -} finally { $p.Dispose() } -if ($failed) { exit 1 }`, opts.Count, psQuote(addr.String()), (opts.Timeout+time.Millisecond-1)/time.Millisecond) +for ($i = 0; $i -lt %d; $i++) { + $r = $p.Send('%s', 2000) + Write-Output ($r.Status.ToString()) + if ($r.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) { $failed = $true } +} +if ($failed) { exit 1 }`, count, addr) return m.Run(ctx, Command{Path: "powershell.exe", Args: []string{"-NoProfile", "-NonInteractive", "-Command", script}}) } - // With both count and deadline, iputils exits nonzero unless all requested - // replies arrive. This matches the Windows IPStatus checks above. - deadline := int(math.Ceil(float64(opts.Count) * (opts.Timeout + time.Second).Seconds())) - args := []string{"-n", "-c", strconv.Itoa(opts.Count), "-W", strconv.FormatFloat(opts.Timeout.Seconds(), 'f', 3, 64), "-w", strconv.Itoa(deadline)} - args = append(args, addr.String()) - return m.Run(ctx, Command{Path: "ping", Args: args}) + return m.Run(ctx, Command{Path: "ping", Args: []string{"-n", "-c", strconv.Itoa(count), "-W", "2", "-w", strconv.Itoa(count * 3), addr.String()}}) } diff --git a/e2e/machine/operations_test.go b/e2e/machine/operations_test.go deleted file mode 100644 index b1ffc6a9..00000000 --- a/e2e/machine/operations_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package machine - -import ( - "context" - "strings" - "testing" - - "github.com/encodeous/nylon/state" -) - -type failedProcess struct { - stopped bool -} - -func (p *failedProcess) Poll(context.Context) (Result, bool, error) { - return Result{Stderr: "Nylon has been initialized\nprocess crashed", ExitCode: 42}, true, nil -} -func (p *failedProcess) Stop(context.Context) error { p.stopped = true; return nil } - -type testMachine struct{ process Process } - -func (m *testMachine) Info() Info { return Info{OS: Linux, WorkDir: "/app/config"} } -func (m *testMachine) Run(context.Context, Command) (Result, error) { return Result{}, nil } -func (m *testMachine) Start(context.Context, Command) (Process, error) { return m.process, nil } -func (m *testMachine) WriteFile(context.Context, string, []byte) error { return nil } -func (m *testMachine) Close(context.Context) error { return nil } - -func TestRunNylonRejectsExitedProcessEvenWithInitializationLog(t *testing.T) { - p := &failedProcess{} - m := &testMachine{process: p} - _, err := RunNylon(t.Context(), m, state.CentralCfg{}, state.LocalCfg{}, NylonOptions{}) - if err == nil || !strings.Contains(err.Error(), "42") { - t.Fatalf("expected startup exit failure, got %v", err) - } - if !p.stopped { - t.Fatal("failed startup process was not cleaned up") - } -} diff --git a/e2e/machine/qemu_linux.go b/e2e/machine/qemu_linux.go index 68afe666..b7234871 100644 --- a/e2e/machine/qemu_linux.go +++ b/e2e/machine/qemu_linux.go @@ -2,11 +2,7 @@ package machine import ( "context" - "crypto/rand" - "encoding/hex" - "errors" "fmt" - "io" "net" "net/netip" "os" @@ -16,96 +12,44 @@ import ( "strings" "syscall" "time" - - "github.com/moby/moby/api/types/container" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" ) +type WindowsConfig struct{ IP, Bridge, Dir, Image string } + type QEMUWindows struct { info Info port int - cfg WindowsConfig dir, tap string cmd *exec.Cmd - done chan struct{} - waitErr error log *os.File } -// NewWindows boots an isolated UEFI guest. Its TAP joins the test bridge; a -// separate user-mode management NIC forwards WinRM to localhost. Neither test -// routes nor link failures can strand the management connection. -// BaseImage must contain the standalone qcow2 prepared by preflight. +// The CI host provides Docker, KVM and Ubuntu's OVMF 4M firmware. func NewWindows(ctx context.Context, cfg WindowsConfig) (_ *QEMUWindows, err error) { - addr, err := netip.ParseAddr(cfg.IP) - if err != nil || !addr.Is4() { - return nil, fmt.Errorf("Windows machine requires a valid IPv4 data address") - } - prefix, err := netip.ParsePrefix(cfg.Subnet) - if err != nil || !prefix.Contains(addr) { - return nil, fmt.Errorf("Windows data IP must belong to the test subnet") - } - if cfg.Bridge == "" || cfg.HostHelperImage == "" { - return nil, fmt.Errorf("Windows requires a local Docker bridge and host helper image") - } - if cfg.Accelerator != "kvm" && cfg.Accelerator != "tcg" { - return nil, fmt.Errorf("QEMU accelerator must be kvm or tcg") - } - if cfg.MemoryMiB < 1 || cfg.CPUs < 1 || cfg.BootTimeout <= 0 { - return nil, fmt.Errorf("invalid Windows VM resources or boot timeout") - } - if _, err := exec.LookPath("qemu-system-x86_64"); err != nil { - return nil, err - } - if cfg.Accelerator == "kvm" { - kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - if err != nil { - return nil, fmt.Errorf("KVM unavailable (use NYLON_E2E_QEMU_ACCEL=tcg explicitly for emulation): %w", err) - } - kvm.Close() - } - if cfg.BaseImage == "" { - return nil, fmt.Errorf("Windows base image must be prepared by preflight") - } - if err := os.MkdirAll(cfg.WorkDir, 0700); err != nil { + ip := netip.MustParseAddr(cfg.IP).As4() + if err := os.MkdirAll(cfg.Dir, 0755); err != nil { return nil, err } - dir, err := os.MkdirTemp(cfg.WorkDir, "windows-") - if err != nil { - return nil, err - } - w := &QEMUWindows{cfg: cfg, dir: dir, info: Info{OS: Windows, IP: cfg.IP, WorkDir: `C:\nylon-e2e`}} + w := &QEMUWindows{dir: cfg.Dir, tap: fmt.Sprintf("ny-%x", ip), info: Info{OS: Windows, IP: cfg.IP, WorkDir: `C:\nylon-e2e`}} defer func() { if err != nil { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() - err = errors.Join(err, w.Close(cleanupCtx)) + _ = w.Close(context.Background()) } }() - overlay := filepath.Join(dir, "disk.qcow2") - if err := CreateOverlay(ctx, cfg.BaseImage, overlay); err != nil { - return nil, err + overlay := filepath.Join(cfg.Dir, "disk.qcow2") + if out, err := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", cfg.Image, overlay).CombinedOutput(); err != nil { + return nil, fmt.Errorf("create overlay: %w: %s", err, out) } - vars, err := os.ReadFile(cfg.UEFIVars) + vars, err := os.ReadFile("/usr/share/OVMF/OVMF_VARS_4M.fd") if err != nil { - return nil, fmt.Errorf("read UEFI variable template: %w", err) - } - varsPath := filepath.Join(dir, "uefi-vars.fd") - if err := os.WriteFile(varsPath, vars, 0600); err != nil { return nil, err } - if _, err := os.Stat(cfg.UEFICode); err != nil { - return nil, fmt.Errorf("read UEFI firmware: %w", err) - } - var random [5]byte - if _, err := rand.Read(random[:]); err != nil { + varsPath := filepath.Join(cfg.Dir, "uefi-vars.fd") + if err := os.WriteFile(varsPath, vars, 0600); err != nil { return nil, err } - w.tap = "ny" + hex.EncodeToString(random[:]) // Linux interface names <= 15 bytes. - macSuffix := fmt.Sprintf("%02x:%02x:%02x", random[0], random[1], random[2]) - dataMAC, mgmtMAC := "52:54:00:"+macSuffix, "52:54:01:"+macSuffix - if err := w.hostNetwork(ctx, `ip tuntap add dev "$1" mode tap user "$3" && ip link set dev "$1" master "$2" && ip link set dev "$1" up`, w.tap, cfg.Bridge, strconv.Itoa(os.Getuid())); err != nil { + dataMAC := fmt.Sprintf("52:54:%02x:%02x:%02x:%02x", ip[0], ip[1], ip[2], ip[3]) + if err := w.hostNetwork(ctx, `ip tuntap add dev "$1" mode tap user "$3"; ip link set dev "$1" master "$2"; ip link set dev "$1" up`, w.tap, cfg.Bridge, strconv.Itoa(os.Getuid())); err != nil { return nil, err } listener, err := net.Listen("tcp4", "127.0.0.1:0") @@ -114,147 +58,69 @@ func NewWindows(ctx context.Context, cfg WindowsConfig) (_ *QEMUWindows, err err } w.port = listener.Addr().(*net.TCPAddr).Port listener.Close() - w.log, err = os.OpenFile(filepath.Join(dir, "qemu.log"), os.O_CREATE|os.O_WRONLY, 0600) + w.log, err = os.Create(filepath.Join(cfg.Dir, "qemu.log")) if err != nil { return nil, err } - cpu := "max" - if cfg.Accelerator == "kvm" { - cpu = "host" - } - args := []string{ - "-name", cfg.Name, "-machine", "q35", "-accel", cfg.Accelerator, "-cpu", cpu, - "-m", strconv.Itoa(cfg.MemoryMiB), "-smp", strconv.Itoa(cfg.CPUs), - "-display", "none", "-monitor", "none", "-serial", "file:" + qemuValue(filepath.Join(dir, "serial.log")), - "-drive", "if=pflash,format=raw,readonly=on,file=" + qemuValue(cfg.UEFICode), - "-drive", "if=pflash,format=raw,file=" + qemuValue(varsPath), - "-drive", "if=ide,format=qcow2,file=" + qemuValue(overlay), - "-netdev", "tap,id=data,ifname=" + w.tap + ",script=no,downscript=no", - "-device", cfg.NICModel + ",netdev=data,mac=" + dataMAC, + cmd := exec.Command("qemu-system-x86_64", + "-machine", "q35", "-accel", "kvm", "-cpu", "host", "-m", "2048", "-smp", "2", + "-display", "none", "-monitor", "none", "-serial", "file:"+filepath.Join(cfg.Dir, "serial.log"), + "-drive", "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd", + "-drive", "if=pflash,format=raw,file="+varsPath, + "-drive", "if=ide,format=qcow2,file="+overlay, + "-netdev", "tap,id=data,ifname="+w.tap+",script=no,downscript=no", + "-device", "virtio-net-pci,netdev=data,mac="+dataMAC, "-netdev", fmt.Sprintf("user,id=mgmt,restrict=on,net=192.168.252.0/24,hostfwd=tcp:127.0.0.1:%d-:5985", w.port), - "-device", cfg.NICModel + ",netdev=mgmt,mac=" + mgmtMAC, - } - w.cmd = exec.Command("qemu-system-x86_64", args...) - // Do not leave VMs running if go test is killed or reaches its hard timeout. - w.cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} - w.cmd.Stdout, w.cmd.Stderr = w.log, w.log - if err := w.cmd.Start(); err != nil { + "-device", "virtio-net-pci,netdev=mgmt,mac=52:54:00:12:34:56") + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + cmd.Stdout, cmd.Stderr = w.log, w.log + if err := cmd.Start(); err != nil { return nil, err } - w.done = make(chan struct{}) - go func() { w.waitErr = w.cmd.Wait(); close(w.done) }() - bootCtx, cancel := context.WithTimeout(ctx, cfg.BootTimeout) + w.cmd = cmd + bootCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - var lastErr error for { - select { - case <-w.done: - return nil, fmt.Errorf("QEMU exited during boot: %v (see %s)", w.waitErr, filepath.Join(dir, "qemu.log")) - default: - } - probeCtx, probeCancel := context.WithTimeout(bootCtx, 20*time.Second) - _, lastErr = w.runPS(probeCtx, "[Console]::Out.Write('ready')", "") - probeCancel() - if lastErr == nil { + if _, err := w.runPS(bootCtx, "'ready'", ""); err == nil { break } if err := pause(bootCtx, time.Second); err != nil { - return nil, fmt.Errorf("Windows WinRM readiness: %w; last error: %v (see %s)", err, lastErr, filepath.Join(dir, "qemu.log")) + return nil, err } } - // Select the data adapter by MAC, never by an unstable Windows adapter name. - // It has no default gateway, leaving management replies on the management NIC. - mac := strings.ReplaceAll(dataMAC, ":", "") - script := `$adapter = Get-NetAdapter | Where-Object { ($_.MacAddress -replace '[:-]', '') -eq ` + psQuote(mac) + ` } -if (!$adapter) { throw 'QEMU test data adapter not found' } -$idx = $adapter.ifIndex -Set-NetIPInterface -InterfaceIndex $idx -AddressFamily IPv4 -Dhcp Disabled -Get-NetIPAddress -InterfaceIndex $idx -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false -ErrorAction SilentlyContinue -New-NetIPAddress -InterfaceIndex $idx -IPAddress ` + psQuote(cfg.IP) + ` -PrefixLength ` + strconv.Itoa(prefix.Bits()) + ` | Out-Null -New-NetFirewallRule -DisplayName 'Nylon e2e data network' -Direction Inbound -InterfaceAlias $adapter.Name -Action Allow -Profile Any | Out-Null -New-NetFirewallRule -DisplayName 'Nylon e2e data outbound' -Direction Outbound -InterfaceAlias $adapter.Name -Action Allow -Profile Any | Out-Null -[void][IO.Directory]::CreateDirectory(` + psQuote(w.info.WorkDir) + `)` + script := `$adapter = Get-NetAdapter | Where-Object { ($_.MacAddress -replace '[:-]', '') -eq ` + psQuote(strings.ReplaceAll(dataMAC, ":", "")) + ` } +Set-NetIPInterface -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -Dhcp Disabled +Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false +New-NetIPAddress -InterfaceIndex $adapter.ifIndex -IPAddress ` + psQuote(cfg.IP) + ` -PrefixLength 24 | Out-Null +Set-NetFirewallProfile -Profile Domain,Private,Public -Enabled False +[void][IO.Directory]::CreateDirectory('C:\nylon-e2e')` if _, err := w.runPS(bootCtx, script, ""); err != nil { - return nil, fmt.Errorf("configure Windows test network: %w", err) + return nil, err } return w, nil } -func qemuValue(s string) string { return strings.ReplaceAll(s, ",", ",,") } - func (w *QEMUWindows) Info() Info { return w.info } -// A local privileged Docker helper configures the host TAP without requiring -// sudo in the Go test runner. The existing debug image already has iproute2. func (w *QEMUWindows) hostNetwork(ctx context.Context, script string, args ...string) error { - c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: w.cfg.HostHelperImage, - Entrypoint: []string{"sh", "-c", script, "machine"}, - Cmd: args, - HostConfigModifier: func(h *container.HostConfig) { h.NetworkMode = "host"; h.Privileged = true }, - WaitingFor: wait.ForExit().WithExitTimeout(30 * time.Second), - }, Started: true, - }) - if c != nil { - defer func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - _ = c.Terminate(cleanupCtx) - }() - } + argv := append([]string{"run", "--rm", "--privileged", "--network", "host", "--entrypoint", "sh", "nylon-debug:latest", "-ec", script, "machine"}, args...) + out, err := exec.CommandContext(ctx, "docker", argv...).CombinedOutput() if err != nil { - return fmt.Errorf("host network helper: %w", err) - } - state, err := c.State(ctx) - if err != nil { - return err - } - if state.ExitCode != 0 { - logs, logErr := c.Logs(ctx) - var output []byte - if logErr == nil { - defer logs.Close() - output, _ = io.ReadAll(logs) - } - return fmt.Errorf("host network helper exited %d: %s", state.ExitCode, output) + return fmt.Errorf("host network: %w: %s", err, out) } return nil } func (w *QEMUWindows) Close(ctx context.Context) error { - var closeErr error - if w.done != nil { - select { - case <-w.done: - default: - // Stop QEMU before removing its writable disk. A canceled cleanup can be retried. - if err := w.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { - closeErr = errors.Join(closeErr, err) - } - select { - case <-w.done: - case <-ctx.Done(): - return errors.Join(closeErr, ctx.Err()) - } - } + if w.cmd != nil { + _ = w.cmd.Process.Kill() + _ = w.cmd.Wait() } if w.log != nil { - closeErr = errors.Join(closeErr, w.log.Close()) - w.log = nil - } - if w.tap != "" { - err := w.hostNetwork(ctx, `if ip link show dev "$1" >/dev/null 2>&1; then ip link delete dev "$1"; fi`, w.tap) - closeErr = errors.Join(closeErr, err) - if err == nil { - w.tap = "" - } - } - // Keep diagnostics, removing only the disposable disk and firmware vars. - for _, name := range []string{"disk.qcow2", "uefi-vars.fd"} { - if err := os.Remove(filepath.Join(w.dir, name)); err != nil && !os.IsNotExist(err) { - closeErr = errors.Join(closeErr, err) - } + _ = w.log.Close() } - return closeErr + err := w.hostNetwork(ctx, `ip link delete dev "$1"`, w.tap) + _ = os.Remove(filepath.Join(w.dir, "disk.qcow2")) + _ = os.Remove(filepath.Join(w.dir, "uefi-vars.fd")) + return err } diff --git a/e2e/machine/qemu_other.go b/e2e/machine/qemu_other.go deleted file mode 100644 index 9d9804d8..00000000 --- a/e2e/machine/qemu_other.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build !linux - -package machine - -import ( - "context" - "fmt" -) - -func NewWindows(context.Context, WindowsConfig) (Machine, error) { - return nil, fmt.Errorf("the QEMU Windows backend requires a local Linux host") -} - -func PrepareWindowsImage(context.Context, WindowsConfig) (string, error) { - return "", fmt.Errorf("the QEMU Windows backend requires a local Linux host") -} diff --git a/e2e/machine/winrm.go b/e2e/machine/winrm.go index 4a26068f..d63033c1 100644 --- a/e2e/machine/winrm.go +++ b/e2e/machine/winrm.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/google/uuid" "github.com/masterzen/winrm" "github.com/masterzen/winrm/soap" ) @@ -32,9 +31,6 @@ func (t *winRMTransport) Transport(endpoint *winrm.Endpoint) error { } func (t *winRMTransport) Post(client *winrm.Client, message *soap.SoapMessage) (string, error) { - if err := t.ctx.Err(); err != nil { - return "", err - } address := net.JoinHostPort(t.endpoint.Host, strconv.Itoa(t.endpoint.Port)) conn, err := (&net.Dialer{Timeout: 20 * time.Second}).DialContext(t.ctx, "tcp", address) if err != nil { @@ -47,11 +43,7 @@ func (t *winRMTransport) Post(client *winrm.Client, message *soap.SoapMessage) ( if err := transport.Transport(t.endpoint); err != nil { return "", err } - response, err := transport.Post(client, message) - if t.ctx.Err() != nil { - return "", t.ctx.Err() - } - return response, err + return transport.Post(client, message) } // runPS uses an encoded command, so scripts never pass through cmd.exe quoting. @@ -59,16 +51,15 @@ func (w *QEMUWindows) runPS(ctx context.Context, script string, input string) (R script = `$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue'; [Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false) $global:LASTEXITCODE = 0 -try { ` + script + ` -exit $LASTEXITCODE -} catch { [Console]::Error.WriteLine($_.ToString()); exit 1 }` +` + script + ` +exit $LASTEXITCODE` ctx, cancel := context.WithCancel(ctx) defer cancel() params := *winrm.DefaultParameters params.Timeout = "PT10S" params.TransportDecorator = func() winrm.Transporter { return &winRMTransport{ctx: ctx} } endpoint := winrm.NewEndpoint("127.0.0.1", w.port, false, false, nil, nil, nil, 20*time.Second) - client, err := winrm.NewClientWithParameters(endpoint, w.cfg.Username, w.cfg.Password, ¶ms) + client, err := winrm.NewClientWithParameters(endpoint, "Administrator", "nylon", ¶ms) if err != nil { return Result{}, err } @@ -76,65 +67,12 @@ exit $LASTEXITCODE return (Result{Stdout: stdout, Stderr: stderr, ExitCode: code}).check(err) } -// windowsArg implements the CommandLineToArgvW/C runtime quoting convention. -func windowsArg(s string) string { - var b strings.Builder - b.WriteByte('"') - slashes := 0 - for _, c := range s { - if c == '\\' { - slashes++ - continue - } - if c == '"' { - b.WriteString(strings.Repeat(`\`, slashes*2+1)) - } else { - b.WriteString(strings.Repeat(`\`, slashes)) - } - slashes = 0 - b.WriteRune(c) - } - b.WriteString(strings.Repeat(`\`, slashes*2)) - b.WriteByte('"') - return b.String() -} - -func windowsArgs(args []string) string { - quoted := make([]string, len(args)) - for i, arg := range args { - quoted[i] = windowsArg(arg) - } - return strings.Join(quoted, " ") -} - func (w *QEMUWindows) Run(ctx context.Context, cmd Command) (Result, error) { - if cmd.Path == "" { - return Result{}, fmt.Errorf("empty executable") + args := []string{psQuote(cmd.Path)} + for _, arg := range cmd.Args { + args = append(args, psQuote(arg)) } - script := `$si = New-Object System.Diagnostics.ProcessStartInfo -$si.UseShellExecute = $false -$si.RedirectStandardOutput = $true -$si.RedirectStandardError = $true -$si.CreateNoWindow = $true -$si.StandardOutputEncoding = New-Object System.Text.UTF8Encoding($false) -$si.StandardErrorEncoding = New-Object System.Text.UTF8Encoding($false) -$si.FileName = ` + psQuote(cmd.Path) + ` -$si.Arguments = ` + psQuote(windowsArgs(cmd.Args)) + "\n" - if cmd.Dir != "" { - script += "$si.WorkingDirectory = " + psQuote(cmd.Dir) + "\n" - } - script += `$p = New-Object System.Diagnostics.Process -$p.StartInfo = $si -try { - [void]$p.Start() - $stdout = $p.StandardOutput.ReadToEndAsync() - $stderr = $p.StandardError.ReadToEndAsync() - $p.WaitForExit() - [Console]::Out.Write($stdout.Result) - [Console]::Error.Write($stderr.Result) - exit $p.ExitCode -} finally { $p.Dispose() }` - return w.runPS(ctx, script, "") + return w.runPS(ctx, "& "+strings.Join(args, " "), "") } // WriteFile sends base64 lines through one WinRM shell. This avoids a separate @@ -149,18 +87,13 @@ func (w *QEMUWindows) WriteFile(ctx context.Context, path string, data []byte) e data = data[n:] } script := `$path = ` + psQuote(path) + ` -[void][IO.Directory]::CreateDirectory([IO.Path]::GetDirectoryName($path)) -$file = [IO.File]::Open($path, [IO.FileMode]::Create, [IO.FileAccess]::Write) -try { - while ($null -ne ($line = [Console]::ReadLine())) { - $bytes = [Convert]::FromBase64String($line) - $file.Write($bytes, 0, $bytes.Length) - } -} finally { $file.Dispose() } -$sha = [Security.Cryptography.SHA256]::Create() -$stream = [IO.File]::OpenRead($path) -try { [Console]::Out.Write([BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '')) } -finally { $stream.Dispose(); $sha.Dispose() }` +$file = [IO.File]::Create($path) +while ($null -ne ($line = [Console]::ReadLine())) { + $bytes = [Convert]::FromBase64String($line) + $file.Write($bytes, 0, $bytes.Length) +} +$file.Close() +(Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash` r, err := w.runPS(ctx, script, input.String()) if err != nil { return err @@ -171,77 +104,44 @@ finally { $stream.Dispose(); $sha.Dispose() }` return nil } -// Shared reads are necessary while the scheduled process is writing its logs. -const psReadShared = `function Read-SharedFile($path) { - $file = [IO.File]::Open($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) - $memory = New-Object IO.MemoryStream - try { $file.CopyTo($memory); return [Convert]::ToBase64String($memory.ToArray()) } - finally { $file.Dispose(); $memory.Dispose() } -} -` - // A scheduled SYSTEM task detaches the process from WinRM's job object. A plain // Start-Process inside a remote shell would be terminated when that shell closes. // SYSTEM also gives Nylon the privileges required by its Windows UAPI listener. func (w *QEMUWindows) Start(ctx context.Context, cmd Command) (Process, error) { - if cmd.Path == "" { - return nil, fmt.Errorf("empty executable") - } - task := "nylon-e2e-" + uuid.NewString() - dir := guestPath(w, task) - stdout, stderr := dir+`\stdout`, dir+`\stderr` - workDir := cmd.Dir - if workDir == "" { - workDir = w.Info().WorkDir - } - argumentList := "" - if len(cmd.Args) != 0 { - argumentList = " -ArgumentList " + psQuote(windowsArgs(cmd.Args)) - } runner := `$ErrorActionPreference = 'Stop' -try { - $p = Start-Process -FilePath ` + psQuote(cmd.Path) + argumentList + ` -WorkingDirectory ` + psQuote(workDir) + ` -RedirectStandardOutput ` + psQuote(stdout) + ` -RedirectStandardError ` + psQuote(stderr) + ` -PassThru - $handle = $p.Handle - [IO.File]::WriteAllText(` + psQuote(dir+`\pid`) + `, [string]$p.Id) - $p.WaitForExit() - [IO.File]::WriteAllText(` + psQuote(dir+`\exit`) + `, [string]$p.ExitCode) -} catch { - [IO.File]::AppendAllText(` + psQuote(stderr) + `, $_.ToString()) - [IO.File]::WriteAllText(` + psQuote(dir+`\exit`) + `, '127') -}` - if err := w.WriteFile(ctx, dir+`\run.ps1`, []byte(runner)); err != nil { +$p = Start-Process -FilePath ` + psQuote(cmd.Path) + ` -ArgumentList ` + psQuote(strings.Join(cmd.Args, " ")) + ` -WorkingDirectory 'C:\nylon-e2e' -RedirectStandardOutput 'C:\nylon-e2e\stdout' -RedirectStandardError 'C:\nylon-e2e\stderr' -PassThru +$handle = $p.Handle +[IO.File]::WriteAllText('C:\nylon-e2e\pid', [string]$p.Id) +$p.WaitForExit() +[IO.File]::WriteAllText('C:\nylon-e2e\exit', [string]$p.ExitCode)` + if err := w.WriteFile(ctx, `C:\nylon-e2e\run.ps1`, []byte(runner)); err != nil { return nil, err } - script := `[IO.File]::WriteAllText(` + psQuote(stdout) + `, '') -[IO.File]::WriteAllText(` + psQuote(stderr) + `, '') -$action = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument ` + psQuote(`-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `+windowsArg(dir+`\run.ps1`)) + ` + script := `[IO.File]::WriteAllText('C:\nylon-e2e\stdout', '') +[IO.File]::WriteAllText('C:\nylon-e2e\stderr', '') +$action = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\nylon-e2e\run.ps1' $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest -$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit ([TimeSpan]::Zero) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -Register-ScheduledTask -TaskName ` + psQuote(task) + ` -Action $action -Principal $principal -Settings $settings | Out-Null -Start-ScheduledTask -TaskName ` + psQuote(task) + ` -for ($i = 0; $i -lt 150; $i++) { - if ((Test-Path -LiteralPath ` + psQuote(dir+`\pid`) + `) -or (Test-Path -LiteralPath ` + psQuote(dir+`\exit`) + `)) { exit 0 } - Start-Sleep -Milliseconds 100 -} -throw 'Scheduled process did not start within 15 seconds'` +Register-ScheduledTask -TaskName 'nylon-e2e' -Action $action -Principal $principal | Out-Null +Start-ScheduledTask -TaskName 'nylon-e2e'` if _, err := w.runPS(ctx, script, ""); err != nil { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - _ = (&windowsProcess{machine: w, dir: dir, task: task}).Stop(cleanupCtx) return nil, err } - return &windowsProcess{machine: w, dir: dir, task: task}, nil + return &windowsProcess{machine: w}, nil } -type windowsProcess struct { - machine *QEMUWindows - dir, task string -} +type windowsProcess struct{ machine *QEMUWindows } func (p *windowsProcess) Poll(ctx context.Context) (Result, bool, error) { - script := psReadShared + `$code = $null -if (Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `) { $code = [int][IO.File]::ReadAllText(` + psQuote(p.dir+`\exit`) + `) } -@{stdout = (Read-SharedFile ` + psQuote(p.dir+`\stdout`) + `); stderr = (Read-SharedFile ` + psQuote(p.dir+`\stderr`) + `); code = $code} | ConvertTo-Json -Compress` + script := `function Read-SharedFile($path) { + $file = [IO.File]::Open($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + $memory = New-Object IO.MemoryStream + $file.CopyTo($memory) + $file.Close() + [Convert]::ToBase64String($memory.ToArray()) +} +$code = $null +if (Test-Path 'C:\nylon-e2e\exit') { $code = [int][IO.File]::ReadAllText('C:\nylon-e2e\exit') } +@{stdout = (Read-SharedFile 'C:\nylon-e2e\stdout'); stderr = (Read-SharedFile 'C:\nylon-e2e\stderr'); code = $code} | ConvertTo-Json -Compress` r, err := p.machine.runPS(ctx, script, "") if err != nil { return Result{}, false, err @@ -250,13 +150,8 @@ if (Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `) { $code = [int][IO.Fi } func (p *windowsProcess) Stop(ctx context.Context) error { - script := `if (!(Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `) -and (Test-Path -LiteralPath ` + psQuote(p.dir+`\pid`) + `)) { - $child = [int][IO.File]::ReadAllText(` + psQuote(p.dir+`\pid`) + `) - Stop-Process -Id $child -Force -ErrorAction SilentlyContinue -} -Stop-ScheduledTask -TaskName ` + psQuote(p.task) + ` -ErrorAction SilentlyContinue -Unregister-ScheduledTask -TaskName ` + psQuote(p.task) + ` -Confirm:$false -ErrorAction SilentlyContinue -if (!(Test-Path -LiteralPath ` + psQuote(p.dir+`\exit`) + `)) { [IO.File]::WriteAllText(` + psQuote(p.dir+`\exit`) + `, ` + psQuote(strconv.Itoa(-1)) + `) }` - _, err := p.machine.runPS(ctx, script, "") + _, err := p.machine.runPS(ctx, `Stop-Process -Id ([int][IO.File]::ReadAllText('C:\nylon-e2e\pid')) -Force +Unregister-ScheduledTask -TaskName 'nylon-e2e' -Confirm:$false +[IO.File]::WriteAllText('C:\nylon-e2e\exit', '-1')`, "") return err } diff --git a/e2e/machine_test.go b/e2e/machine_test.go index 937404d4..e0dbcaf3 100644 --- a/e2e/machine_test.go +++ b/e2e/machine_test.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e && linux package e2e @@ -43,7 +43,7 @@ func TestMachines(t *testing.T) { h.WaitForStatus(t, peer.name, func(status *protocol.StatusResponse) bool { return HasSelectedRoute(status, peer.destination+"/32", peer.nextHop, peer.nextHop) }) - result, err := h.Ping(peer.name, netip.MustParseAddr(peer.destination), machine.PingOptions{Count: 3}) + result, err := h.Ping(peer.name, netip.MustParseAddr(peer.destination), 3) if err != nil { h.PrintLogs(peer.name) t.Fatalf("%s ping: %v\n%s\n%s", peer.name, err, result.Stdout, result.Stderr) diff --git a/e2e/machines.go b/e2e/machines.go index 655ecd94..a371889f 100644 --- a/e2e/machines.go +++ b/e2e/machines.go @@ -1,32 +1,24 @@ -//go:build e2e +//go:build e2e && linux package e2e import ( - "fmt" + "errors" "net/netip" "os" - "strings" + "path/filepath" "sync" "time" "github.com/encodeous/nylon/e2e/machine" "github.com/encodeous/nylon/state" "github.com/goccy/go-yaml" - "github.com/testcontainers/testcontainers-go" ) // StartMachines is the portable replacement for StartNodes. Existing fixtures // may continue using StartNodes/Nodes until their OS-specific operations migrate. func (h *Harness) StartMachines(specs ...NodeSpec) { h.t.Helper() - names := make(map[string]bool) - ips := make(map[string]bool) - h.mu.Lock() - for name := range h.Machines { - names[name] = true - } - h.mu.Unlock() needsWindows := false for i := range specs { spec := &specs[i] @@ -36,44 +28,26 @@ func (h *Harness) StartMachines(specs ...NodeSpec) { spec.OS = machine.Linux } } - if spec.OS != machine.Linux && spec.OS != machine.Windows { - h.t.Fatalf("unsupported machine OS %q (use linux or windows)", spec.OS) - } needsWindows = needsWindows || spec.OS == machine.Windows - if spec.Name == "" || names[spec.Name] { - h.t.Fatalf("empty or duplicate machine name %q", spec.Name) - } - names[spec.Name] = true - if spec.IP != "" { - if ips[spec.IP] { - h.t.Fatalf("duplicate machine data IP %q", spec.IP) - } - ips[spec.IP] = true - } } + h.t.Log("Running machine preflight") - prepared, err := h.preflight(needsWindows) - if err != nil { - h.t.Fatalf("machine preflight: %v", err) + var prepared windowsPrerequisites + if needsWindows { + var err error + prepared, err = preflight(h.ctx, h.RootDir) + if err != nil { + h.t.Fatal(err) + } } - errs := make(chan error, len(specs)) + errs := make([]error, len(specs)) var wg sync.WaitGroup - for _, spec := range specs { - wg.Add(1) - go func() { - defer wg.Done() - if err := h.startMachine(spec, prepared); err != nil { - errs <- fmt.Errorf("machine %s: %w", spec.Name, err) - } - }() + for i, spec := range specs { + wg.Go(func() { errs[i] = h.startMachine(spec, prepared) }) } wg.Wait() - close(errs) - for err := range errs { - h.t.Error(err) - } - if h.t.Failed() { - h.t.FailNow() + if err := errors.Join(errs...); err != nil { + h.t.Fatal(err) } } @@ -86,17 +60,13 @@ func (h *Harness) startMachine(spec NodeSpec, prepared windowsPrerequisites) err h.t.Logf("Starting %s machine %s at %s", osType, spec.Name, spec.IP) switch osType { case machine.Linux: - m, err = machine.NewDocker(h.ctx, machine.DockerConfig{Name: spec.Name, Image: ImageName, Network: h.Network.Name, IP: spec.IP}) + m, err = machine.NewDocker(h.ctx, machine.DockerConfig{Name: spec.Name, Network: h.Network.Name, IP: spec.IP}) case machine.Windows: - addr, parseErr := netip.ParseAddr(spec.IP) - if parseErr != nil || !addr.Is4() || addr.As4()[3] < 2 || addr.As4()[3] >= 128 { - return fmt.Errorf("Windows requires an explicit data IP in hosts .2 through .127") - } - cfg := prepared.config opts.Binary, opts.WintunDLL = prepared.binary, prepared.dll - cfg.Name, cfg.IP = spec.Name, spec.IP - m, err = machine.NewWindows(h.ctx, cfg) - opts.StartupTimeout = time.Minute + m, err = machine.NewWindows(h.ctx, machine.WindowsConfig{ + IP: spec.IP, Bridge: "br-" + h.Network.ID[:12], Image: prepared.image, + Dir: filepath.Join(h.RootDir, "workdir", "e2e", "vms", h.t.Name(), spec.Name), + }) } if err != nil { return err @@ -139,32 +109,18 @@ func (h *Harness) followProcess(name string, p machine.Process) { go func() { defer h.logWG.Done() var offsets [2]int - consumer := UnifiedLogConsumer{Node: name, Manager: h.LogManager} + writers := [2]managerWriter{} + for i := range writers { + writers[i] = managerWriter{node: name, source: SourceStdout, manager: h.LogManager} + } for { r, exited, err := p.Poll(h.ctx) if err != nil { - if h.ctx.Err() == nil { - h.t.Logf("read logs for %s: %v", name, err) - } return } for i, data := range []string{r.Stdout, r.Stderr} { - chunk := data[offsets[i]:] - end := strings.LastIndexByte(chunk, '\n') + 1 - if exited { - end = len(chunk) - } - // Preserve per-line matching, including Windows logs on stderr. - for _, line := range strings.SplitAfter(chunk[:end], "\n") { - if line == "" { - continue - } - consumer.Accept(testcontainers.Log{LogType: "stdout", Content: []byte(line)}) - if i == 1 { - h.LogManager.Accept(name, SourceStderr, StripAnsi(line)) - } - } - offsets[i] += end + _, _ = writers[i].Write([]byte(data[offsets[i]:])) + offsets[i] = len(data) } if exited { return @@ -178,12 +134,9 @@ func (h *Harness) followProcess(name string, p machine.Process) { }() } -func (h *Harness) Ping(name string, addr netip.Addr, opts machine.PingOptions) (machine.Result, error) { +func (h *Harness) Ping(name string, addr netip.Addr, count int) (machine.Result, error) { h.mu.Lock() m := h.Machines[name] h.mu.Unlock() - if m == nil { - return machine.Result{}, fmt.Errorf("machine %q not found", name) - } - return machine.Ping(h.ctx, m, addr, opts) + return machine.Ping(h.ctx, m, addr, count) } diff --git a/e2e/preflight.go b/e2e/preflight.go index dff91bad..e26e531a 100644 --- a/e2e/preflight.go +++ b/e2e/preflight.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e && linux package e2e @@ -15,15 +15,13 @@ import ( "sync" "time" - "github.com/encodeous/nylon/e2e/machine" - "github.com/moby/moby/client" - "github.com/testcontainers/testcontainers-go" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/joho/godotenv" + "github.com/klauspost/compress/zstd" ) -type windowsPrerequisites struct { - config machine.WindowsConfig - binary, dll string -} +type windowsPrerequisites struct{ image, binary, dll string } var windowsPreflight struct { sync.Once @@ -31,132 +29,101 @@ var windowsPreflight struct { err error } -// preflight completes shared preparation before StartMachines launches goroutines. -// Linux scenarios need no Windows artifacts. Explicit Windows NodeSpec overrides -// take this path too, regardless of the default placement. -func (h *Harness) preflight(needsWindows bool) (windowsPrerequisites, error) { - if !needsWindows { - return windowsPrerequisites{}, nil - } - windowsPreflight.Do(func() { - fmt.Println("Preflight: preparing Windows base image and Nylon artifacts...") - cfg, err := machine.WindowsConfigFromEnv(h.RootDir) - if err == nil { - cfg.BaseImage, err = machine.PrepareWindowsImage(h.ctx, cfg) - } - prepared := windowsPrerequisites{config: cfg} - if err == nil { - prepared.binary, prepared.dll, err = prepareWindowsArtifacts(h.ctx, h.RootDir) - } - windowsPreflight.prepared, windowsPreflight.err = prepared, err - }) - if windowsPreflight.err != nil { - return windowsPrerequisites{}, windowsPreflight.err - } - provider, err := testcontainers.NewDockerProvider() - if err != nil { - return windowsPrerequisites{}, err - } - defer provider.Close() - inspection, err := provider.Client().NetworkInspect(h.ctx, h.Network.ID, client.NetworkInspectOptions{}) - if err != nil { - return windowsPrerequisites{}, err - } - prepared := windowsPreflight.prepared - prepared.config.Subnet, prepared.config.HostHelperImage = h.Subnet, ImageName - prepared.config.Bridge = inspection.Network.Options["com.docker.network.bridge.name"] - if prepared.config.Bridge == "" { - prepared.config.Bridge = "br-" + inspection.Network.ID[:12] - } - return prepared, nil +func preflight(ctx context.Context, root string) (windowsPrerequisites, error) { + windowsPreflight.Do(func() { windowsPreflight.prepared, windowsPreflight.err = prepareWindows(ctx, root) }) + return windowsPreflight.prepared, windowsPreflight.err } -func prepareWindowsArtifacts(ctx context.Context, root string) (string, string, error) { - dir := filepath.Join(root, "workdir", "e2e", "artifacts") - if err := os.MkdirAll(dir, 0700); err != nil { - return "", "", err - } - binary := os.Getenv("NYLON_E2E_WINDOWS_BINARY") - if binary != "" && !filepath.IsAbs(binary) { - binary = filepath.Join(root, binary) - } - if binary == "" { - binary = filepath.Join(dir, "nylon.exe") - cmd := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") - cmd.Dir = root - cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=windows", "GOARCH=amd64") - if out, err := cmd.CombinedOutput(); err != nil { - return "", "", fmt.Errorf("build Windows Nylon: %w: %s", err, out) +func prepareWindows(ctx context.Context, root string) (windowsPrerequisites, error) { + dir := filepath.Join(root, "workdir", "e2e") + p := windowsPrerequisites{ + image: filepath.Join(dir, "images", "nylon-ci-win2025.qcow2"), + binary: filepath.Join(dir, "nylon.exe"), + dll: filepath.Join(dir, "artifacts", "wintun-0.14.1-amd64.dll"), + } + if err := os.MkdirAll(filepath.Dir(p.image), 0755); err != nil { + return p, err + } + if _, err := os.Stat(p.image); os.IsNotExist(err) { + _ = godotenv.Load(filepath.Join(root, ".env")) + if err := downloadWindowsImage(ctx, p.image); err != nil { + return p, err } } - if _, err := os.Stat(binary); err != nil { - return "", "", err + cmd := exec.CommandContext(ctx, "go", "build", "-o", p.binary, ".") + cmd.Dir = root + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=windows", "GOARCH=amd64") + if out, err := cmd.CombinedOutput(); err != nil { + return p, fmt.Errorf("build Windows Nylon: %w: %s", err, out) } - dll := os.Getenv("NYLON_E2E_WINTUN_DLL") - if dll != "" { - if !filepath.IsAbs(dll) { - dll = filepath.Join(root, dll) + if _, err := os.Stat(p.dll); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(p.dll), 0755); err != nil { + return p, err } - if _, err := os.Stat(dll); err != nil { - return "", "", err + resp, err := http.Get("https://www.wintun.net/builds/wintun-0.14.1.zip") + if err != nil { + return p, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return p, err + } + archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return p, err + } + dll, err := archive.Open("wintun/bin/amd64/wintun.dll") + if err != nil { + return p, err + } + defer dll.Close() + data, err = io.ReadAll(dll) + if err != nil { + return p, err + } + if err := os.WriteFile(p.dll, data, 0644); err != nil { + return p, err } - return binary, dll, nil - } - // Match the version and architecture already used by the release workflow. - dll = filepath.Join(dir, "wintun-0.14.1-amd64.dll") - if _, err := os.Stat(dll); err == nil { - return binary, dll, nil } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.wintun.net/builds/wintun-0.14.1.zip", nil) + return p, nil +} + +func downloadWindowsImage(ctx context.Context, path string) error { + const payloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + url := os.Getenv("AWS_ENDPOINT_URL_S3") + "/nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return "", "", err + return err + } + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + credentials := aws.Credentials{AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY")} + if err := v4.NewSigner().SignHTTP(ctx, credentials, req, payloadHash, "s3", "auto", time.Now()); err != nil { + return err } - resp, err := (&http.Client{Timeout: 2 * time.Minute}).Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { - return "", "", err + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", "", fmt.Errorf("download Wintun: HTTP %d", resp.StatusCode) + return fmt.Errorf("download Windows image: HTTP %d", resp.StatusCode) } - data, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + decoder, err := zstd.NewReader(resp.Body) if err != nil { - return "", "", err + return err } - archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + defer decoder.Close() + file, err := os.Create(path + ".partial") if err != nil { - return "", "", err + return err } - for _, entry := range archive.File { - if entry.Name != "wintun/bin/amd64/wintun.dll" { - continue - } - reader, err := entry.Open() - if err != nil { - return "", "", err - } - defer reader.Close() - data, err := io.ReadAll(io.LimitReader(reader, 16<<20)) - if err != nil { - return "", "", err - } - tmp, err := os.CreateTemp(dir, ".wintun-*.dll") - if err != nil { - return "", "", err - } - defer os.Remove(tmp.Name()) - _, writeErr := tmp.Write(data) - closeErr := tmp.Close() - if writeErr != nil { - return "", "", writeErr - } - if closeErr != nil { - return "", "", closeErr - } - if err := os.Rename(tmp.Name(), dll); err != nil { - return "", "", err - } - return binary, dll, nil + defer file.Close() + if _, err := io.Copy(file, decoder); err != nil { + return err + } + if err := file.Close(); err != nil { + return err } - return "", "", fmt.Errorf("Wintun archive has no amd64 DLL") + return os.Rename(file.Name(), path) } diff --git a/go.mod b/go.mod index 27f40c89..7748362f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/gaissmai/bart v0.25.0 github.com/goccy/go-yaml v1.19.1 github.com/google/go-cmp v0.7.0 - github.com/google/uuid v1.6.0 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.18.7 @@ -59,6 +58,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect From e9b10457ec1593e149192fe6712ba5740a7bc7fe Mon Sep 17 00:00:00 2001 From: Dongruixuan Li Date: Sun, 1 Nov 2026 20:20:28 -0500 Subject: [PATCH 3/4] refactor(e2e): isolate Windows connectivity and build its image in CI --- .github/workflows/go-test.yml | 59 +++++++++++ core/sys_route_test.go | 72 ++----------- e2e/README.md | 40 -------- e2e/connectivity_test.go | 69 ++++--------- e2e/harness.go | 66 +----------- e2e/machine/docker.go | 113 --------------------- e2e/machine/machine.go | 100 ------------------ e2e/machine/operations.go | 111 -------------------- e2e/machine/qemu_linux.go | 126 ----------------------- e2e/machine/winrm.go | 157 ---------------------------- e2e/machine_test.go | 71 ------------- e2e/machines.go | 142 -------------------------- e2e/preflight.go | 129 ----------------------- e2e/windows/README.md | 31 ++++++ e2e/windows/connectivity.py | 186 ++++++++++++++++++++++++++++++++++ e2e/windows/run.sh | 14 +++ go.mod | 17 +--- go.sum | 77 -------------- 18 files changed, 322 insertions(+), 1258 deletions(-) delete mode 100644 e2e/README.md delete mode 100644 e2e/machine/docker.go delete mode 100644 e2e/machine/machine.go delete mode 100644 e2e/machine/operations.go delete mode 100644 e2e/machine/qemu_linux.go delete mode 100644 e2e/machine/winrm.go delete mode 100644 e2e/machine_test.go delete mode 100644 e2e/machines.go delete mode 100644 e2e/preflight.go create mode 100644 e2e/windows/README.md create mode 100644 e2e/windows/connectivity.py create mode 100755 e2e/windows/run.sh diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index d0e88f4f..ee52da2c 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -75,3 +75,62 @@ jobs: env: CGO_ENABLED: "0" run: go run gotest.tools/gotestsum@latest -- -count=1 -tags=e2e ./e2e/... -run '^TestConnectivity$' + e2e_windows: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install Windows e2e dependencies + run: | + # Leave room for the installation ISOs, qcow2 and compressed Nix output. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf python3-venv unzip zstd + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + fallback = true + connect-timeout = 5 + download-attempts = 1 + extra-system-features = kvm + - name: Build Windows image + id: windows_image + env: + AWS_ENDPOINT_URL_S3: ${{ secrets.AWS_ENDPOINT_URL_S3 }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PUBLISH_CACHE: ${{ github.event_name == 'push' }} + run: | + cache_args=() + if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then + # Both the Nix daemon (reads) and nix copy (writes) need credentials. + mkdir -p "$HOME/.aws" + printf '[default]\naws_access_key_id=%s\naws_secret_access_key=%s\n' \ + "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" > "$HOME/.aws/credentials" + chmod 600 "$HOME/.aws/credentials" + sudo mkdir -p /root/.aws + sudo install -m 600 "$HOME/.aws/credentials" /root/.aws/credentials + # This private, authenticated bucket is trusted directly. + cache="s3://nylon-ci-windows-image?endpoint=${AWS_ENDPOINT_URL_S3#https://}®ion=auto&trusted=true" + cache_args=(--option extra-substituters "$cache") + fi + nix build github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025 \ + --out-link "$RUNNER_TEMP/windows-image" --fallback "${cache_args[@]}" + if [[ -n "$AWS_ACCESS_KEY_ID" && "$PUBLISH_CACHE" == true ]]; then + # The output contains an already compressed qcow2; don't recompress it. + nix copy --no-recursive --to "$cache&compression=none&multipart-upload=true" "$RUNNER_TEMP/windows-image" \ + || echo '::warning::Could not upload Windows image to the Nix cache' + fi + zstd -d "$RUNNER_TEMP/windows-image/nylon-ci-win2025.qcow2.zst" -o "$RUNNER_TEMP/windows.qcow2" + echo "qcow2=$RUNNER_TEMP/windows.qcow2" >> "$GITHUB_OUTPUT" + - name: Run Windows e2e + run: e2e/windows/run.sh '${{ steps.windows_image.outputs.qcow2 }}' + - name: Upload Windows e2e logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: windows-e2e-logs + path: workdir/e2e/windows/**/*.log diff --git a/core/sys_route_test.go b/core/sys_route_test.go index fc3391f3..44c18830 100644 --- a/core/sys_route_test.go +++ b/core/sys_route_test.go @@ -95,69 +95,15 @@ func TestComputeSysRouteTableCoalescesAdjacentResults(t *testing.T) { assert.Equal(t, want, sortedPrefixes(n.ComputeSysRouteTable())) } -func TestSystemRoutePrefixesPlatformAggregation(t *testing.T) { - for _, tc := range []struct { - name string - prefixes, excluded []netip.Prefix - merged, separate []netip.Prefix - }{ - { - name: "adjacent hosts", - prefixes: []netip.Prefix{pfx("10.0.0.3/32"), pfx("10.0.0.2/32")}, - merged: []netip.Prefix{pfx("10.0.0.2/31")}, - separate: []netip.Prefix{pfx("10.0.0.2/32"), pfx("10.0.0.3/32")}, - }, - { - name: "four adjacent hosts", - prefixes: []netip.Prefix{pfx("10.0.0.4/32"), pfx("10.0.0.5/32"), pfx("10.0.0.6/32"), pfx("10.0.0.7/32")}, - merged: []netip.Prefix{pfx("10.0.0.4/30")}, - separate: []netip.Prefix{pfx("10.0.0.4/32"), pfx("10.0.0.5/32"), pfx("10.0.0.6/32"), pfx("10.0.0.7/32")}, - }, - { - name: "adjacent subnets", - prefixes: []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")}, - merged: []netip.Prefix{pfx("10.0.0.0/24")}, - separate: []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")}, - }, - { - name: "explicit subnet stays intact", - prefixes: []netip.Prefix{pfx("10.0.0.2/31")}, - merged: []netip.Prefix{pfx("10.0.0.2/31")}, - separate: []netip.Prefix{pfx("10.0.0.2/31")}, - }, - { - name: "IPv6 hosts", - prefixes: []netip.Prefix{pfx("fd00::2/128"), pfx("fd00::3/128")}, - merged: []netip.Prefix{pfx("fd00::2/127")}, - separate: []netip.Prefix{pfx("fd00::2/128"), pfx("fd00::3/128")}, - }, - { - name: "exclusions split an advertised subnet", - prefixes: []netip.Prefix{pfx("10.0.0.0/24")}, - excluded: []netip.Prefix{pfx("10.0.0.64/26")}, - merged: []netip.Prefix{pfx("10.0.0.0/26"), pfx("10.0.0.128/25")}, - separate: []netip.Prefix{pfx("10.0.0.0/26"), pfx("10.0.0.128/25")}, - }, - { - name: "overlap after exclusions is deduplicated", - prefixes: []netip.Prefix{pfx("10.0.0.0/24"), pfx("10.0.0.0/25")}, - excluded: []netip.Prefix{pfx("10.0.0.128/25")}, - merged: []netip.Prefix{pfx("10.0.0.0/25")}, - separate: []netip.Prefix{pfx("10.0.0.0/25")}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - original := slices.Clone(tc.prefixes) - for _, goos := range []string{"linux", "darwin", "windows"} { - want := tc.merged - if goos == "windows" { - want = tc.separate - } - assert.Equal(t, want, systemRoutePrefixes(tc.prefixes, state.MakeSet(tc.excluded), goos), goos) - } - assert.Equal(t, original, tc.prefixes, "selected prefixes must not be mutated") - }) - } +func TestSystemRoutePrefixesWindowsKeepsAdvertisedPrefixes(t *testing.T) { + hosts := []netip.Prefix{pfx("10.0.0.2/32"), pfx("10.0.0.3/32")} + empty := state.MakeSet(nil) + assert.Equal(t, hosts, systemRoutePrefixes(hosts, empty, "windows")) + assert.Equal(t, []netip.Prefix{pfx("10.0.0.2/31")}, systemRoutePrefixes(hosts, empty, "linux")) + + subnets := []netip.Prefix{pfx("10.0.0.0/25"), pfx("10.0.0.128/25")} + assert.Equal(t, subnets, systemRoutePrefixes(subnets, empty, "windows")) + assert.Equal(t, []netip.Prefix{pfx("10.0.0.0/24")}, systemRoutePrefixes(subnets, empty, "linux")) } func sysRouteTestNylon(local state.NodeId, centralExcludes, localUnexcludes, localExcludes []netip.Prefix, routes map[netip.Prefix]state.SelRoute) *Nylon { diff --git a/e2e/README.md b/e2e/README.md deleted file mode 100644 index 95965e07..00000000 --- a/e2e/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Portable e2e machines - -`StartMachines`, `machine.RunNylon` and `Harness.Ping` run the same scenarios on -Linux Docker containers and QEMU Windows guests. `NodeSpec.OS` selects individual -nodes; `NYLON_E2E_MACHINE` selects the default. Auxiliary services and unmigrated -Linux tests keep using `StartNodes`. - -```sh -NYLON_E2E_MACHINE=windows go test -tags=e2e ./e2e \ - -run '^(TestMachines|TestConnectivity)$' -count=1 -parallel=1 -timeout=20m -v -NYLON_E2E_MACHINE=mixed go test -tags=e2e ./e2e \ - -run '^TestConnectivity$' -count=1 -parallel=1 -timeout=20m -v -``` - -The runner is Linux with a local rootful Docker daemon/CLI, writable `/dev/kvm`, -`/dev/net/tun`, `qemu-system-x86_64`, `qemu-img`, and Ubuntu's OVMF 4M firmware. -On Ubuntu install `qemu-system-x86 qemu-utils ovmf`. Guests use fixed settings: -Windows Server 2025, UEFI, virtio NICs, 2 CPUs, 2 GiB RAM and WinRM HTTP/NTLM on -port 5985 as `Administrator` / `nylon`. The image includes the NIC drivers. - -Preflight downloads `nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst` from R2, -builds Windows amd64 Nylon, and prepares Wintun 0.14.1 once before parallel boot. -R2 credentials are `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, and -`AWS_SECRET_ACCESS_KEY`, supplied as environment variables or in the ignored root -`.env`. The image is cached in `workdir/e2e/images`; clear it to replace the image. -Each VM uses a new qcow2 overlay and UEFI variables; cleanup removes both and -retains diagnostics under `workdir/e2e/{vms,logs}`. - -A TAP joins each data NIC to the scenario's `/24` Docker bridge. Docker assigns -addresses from `.128/25`; nodes use explicit addresses in `.2`–`.127`. A separate -restricted management NIC forwards WinRM to localhost and cannot carry test -traffic. Disposable guests have their firewall disabled. Each machine runs one -Nylon process, launched as a SYSTEM task on Windows; uploads use SHA256 checks. - -Windows installs selected prefixes separately to avoid merging adjacent hosts -into on-link subnets with broadcast addresses. Linux/macOS retain aggregation; -explicitly advertised subnets remain subnet routes. - -CI workflow integration will follow separately once Vault credentials are ready. -Allow 4 GiB of guest RAM for two Windows nodes or 6 GiB for three. diff --git a/e2e/connectivity_test.go b/e2e/connectivity_test.go index b34e1f87..4600dfc0 100644 --- a/e2e/connectivity_test.go +++ b/e2e/connectivity_test.go @@ -1,15 +1,11 @@ -//go:build e2e && linux +//go:build e2e package e2e import ( - "net/netip" - "os" "testing" "time" - "github.com/encodeous/nylon/e2e/machine" - "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" ) @@ -62,52 +58,27 @@ func TestConnectivity(t *testing.T) { node3Cfg := SimpleLocal("node3", node3Key) node3Path := h.WriteConfig(configDir, "node3.yaml", node3Cfg) - // The scenario is identical for Linux, Windows, and a Windows relay between - // Linux endpoints. Only the deployment placement changes. - specs := []NodeSpec{ - {Name: "node1", IP: node1IP, CentralConfigPath: centralPath, NodeConfigPath: node1Path}, - {Name: "node2", IP: node2IP, CentralConfigPath: centralPath, NodeConfigPath: node2Path}, - {Name: "node3", IP: node3IP, CentralConfigPath: centralPath, NodeConfigPath: node3Path}, - } - if os.Getenv("NYLON_E2E_MACHINE") == "mixed" { - specs[0].OS, specs[1].OS, specs[2].OS = machine.Linux, machine.Windows, machine.Linux - } - h.StartMachines(specs...) + // 4. Start Containers in Parallel + h.StartNodes( + NodeSpec{Name: "node1", IP: node1IP, CentralConfigPath: centralPath, NodeConfigPath: node1Path}, + NodeSpec{Name: "node2", IP: node2IP, CentralConfigPath: centralPath, NodeConfigPath: node2Path}, + NodeSpec{Name: "node3", IP: node3IP, CentralConfigPath: centralPath, NodeConfigPath: node3Path}, + ) // 5. Wait for convergence t.Log("Waiting for convergence...") - h.WaitForStatus(t, "node3", func(s *protocol.StatusResponse) bool { - return HasSelectedRoute(s, "10.0.0.1/32", "node2", "node1") - }) - h.WaitForStatus(t, "node1", func(s *protocol.StatusResponse) bool { - return HasSelectedRoute(s, "10.0.0.3/32", "node2", "node3") - }) - // Selected control-plane routes can precede system-route installation. - // Wait for the data plane before asserting consecutive successful replies. - deadline := time.Now().Add(WaitTimeout) - for { - result, err := h.Ping("node3", netip.MustParseAddr(node1NylonIP), 1) - if err == nil { - break - } - if time.Now().After(deadline) { - t.Fatalf("data plane did not converge: %v\n%s\n%s", err, result.Stdout, result.Stderr) - } - time.Sleep(500 * time.Millisecond) - } - - // Check both multihop directions and a Windows-originated ping in mixed mode. - for _, peer := range []struct{ from, to string }{ - {"node3", node1NylonIP}, {"node2", node1NylonIP}, {"node1", node3NylonIP}, - } { - t.Logf("Pinging %s from %s...", peer.to, peer.from) - result, err := h.Ping(peer.from, netip.MustParseAddr(peer.to), 3) - if err != nil { - for _, node := range []string{"node1", "node2", "node3"} { - h.PrintLogs(node) - } - t.Fatalf("%s ping to %s: %v\n%s\n%s", peer.from, peer.to, err, result.Stdout, result.Stderr) - } - t.Logf("Ping output:\n%s", result.Stdout) + h.WaitForLog("node3", "installing new route prefix=10.0.0.1/32") + h.WaitForLog("node1", "installing new route prefix=10.0.0.2/31") + + // 6. Test Connectivity + // Ping from node1 to node2's Nylon IP + t.Logf("Pinging %s from node1...", node2NylonIP) + stdout, stderr, err := h.Exec("node3", []string{"ping", "-c", "3", node1NylonIP}) + if err != nil { + h.PrintLogs("node1") + h.PrintLogs("node2") + h.PrintLogs("node3") + t.Fatalf("Ping failed: %v\nStdout: %s\nStderr: %s", err, stdout, stderr) } + t.Logf("Ping output:\n%s", stdout) } diff --git a/e2e/harness.go b/e2e/harness.go index be650841..0cb6fcdd 100644 --- a/e2e/harness.go +++ b/e2e/harness.go @@ -16,7 +16,6 @@ import ( "testing" "time" - "github.com/encodeous/nylon/e2e/machine" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "github.com/goccy/go-yaml" @@ -43,10 +42,6 @@ type Harness struct { ctx context.Context Network *testcontainers.DockerNetwork Nodes map[string]testcontainers.Container - Machines map[string]machine.Machine - processes map[string]machine.Process - logCancel context.CancelFunc - logWG sync.WaitGroup LogManager *LogManager ImageName string RootDir string @@ -82,15 +77,11 @@ func NewHarness(t *testing.T) *Harness { } t.Logf("Allocated subnet: %s, gateway: %s", subnet, gateway) - ctx, logCancel := context.WithCancel(ctx) h := &Harness{ t: t, ctx: ctx, Network: newNetwork, Nodes: make(map[string]testcontainers.Container), - Machines: make(map[string]machine.Machine), - processes: make(map[string]machine.Process), - logCancel: logCancel, LogManager: NewLogManager(), RootDir: rootDir, Subnet: subnet, @@ -126,9 +117,7 @@ func createNetwork(ctx context.Context) (*testcontainers.DockerNetwork, string, Driver: "default", Config: []network.IPAMConfig{ { - Subnet: netip.MustParsePrefix(subnet), - // Dynamic Docker addresses stay clear of static VM addresses. - IPRange: netip.PrefixFrom(netip.MustParseAddr(GetIP(subnet, 128)), 25), + Subnet: netip.MustParsePrefix(subnet), Gateway: netip.MustParseAddr(gateway), }, }, @@ -150,8 +139,6 @@ type NodeSpec struct { CentralConfigPath string NodeConfigPath string ExtraArgs []string - // OS is used by StartMachines. Empty selects NYLON_E2E_MACHINE (linux by default). - OS machine.OS } func (h *Harness) StartNodes(specs ...NodeSpec) { @@ -426,40 +413,10 @@ func (h *Harness) StartTrace(nodeName string) { }() } func (h *Harness) Cleanup() { - h.logCancel() - h.logWG.Wait() h.mu.Lock() defer h.mu.Unlock() - cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - if len(h.processes) != 0 { - dir := filepath.Join(h.RootDir, "workdir", "e2e", "logs", h.t.Name()) - if err := os.MkdirAll(dir, 0700); err != nil { - h.t.Logf("create machine log directory: %v", err) - } else { - for name, p := range h.processes { - r, _, err := p.Poll(cleanupCtx) - if err != nil { - h.t.Logf("save machine logs for %s: %v", name, err) - continue - } - for suffix, content := range map[string]string{"stdout": r.Stdout, "stderr": r.Stderr} { - if err := os.WriteFile(filepath.Join(dir, name+"."+suffix+".log"), []byte(content), 0600); err != nil { - h.t.Logf("save machine log: %v", err) - } - } - } - } - } - for name, m := range h.Machines { - err := m.Close(cleanupCtx) - if err != nil { - h.t.Logf("failed to close machine %s: %v", name, err) - } - } for name, c := range h.Nodes { - err := c.Terminate(cleanupCtx) - if err != nil { + if err := c.Terminate(h.ctx); err != nil { h.t.Logf("failed to terminate container %s: %v", name, err) } } @@ -469,17 +426,8 @@ func (h *Harness) Cleanup() { } func (h *Harness) Exec(nodeName string, cmd []string) (string, string, error) { h.mu.Lock() - m := h.Machines[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() - if m != nil { - path := cmd[0] - if path == "nylon" && m.Info().OS == machine.Windows { - path = m.Info().WorkDir + `\nylon.exe` - } - r, err := m.Run(h.ctx, machine.Command{Path: path, Args: cmd[1:]}) - return StripAnsi(r.Stdout), StripAnsi(r.Stderr), err - } if !ok { return "", "", fmt.Errorf("node %s not found", nodeName) @@ -540,12 +488,8 @@ func (h *Harness) ExecBackground(nodeName string, cmd []string) *BackgroundExec // GetIP returns the IP address of the node in the test network func (h *Harness) GetIP(nodeName string) (string, error) { h.mu.Lock() - m := h.Machines[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() - if m != nil { - return m.Info().IP, nil - } if !ok { return "", fmt.Errorf("node %s not found", nodeName) } @@ -553,14 +497,8 @@ func (h *Harness) GetIP(nodeName string) (string, error) { } func (h *Harness) PrintLogs(nodeName string) { h.mu.Lock() - p := h.processes[nodeName] container, ok := h.Nodes[nodeName] h.mu.Unlock() - if p != nil { - r, _, err := p.Poll(h.ctx) - h.t.Logf("Logs for %s (error: %v):\n%s\n%s", nodeName, err, r.Stdout, r.Stderr) - return - } if !ok { h.t.Logf("node %s not found for logging", nodeName) return diff --git a/e2e/machine/docker.go b/e2e/machine/docker.go deleted file mode 100644 index eb8563d7..00000000 --- a/e2e/machine/docker.go +++ /dev/null @@ -1,113 +0,0 @@ -package machine - -import ( - "bytes" - "context" - "net/netip" - "os/exec" - "strings" - - "github.com/moby/moby/api/types/container" - "github.com/moby/moby/api/types/network" - "github.com/testcontainers/testcontainers-go" -) - -type DockerConfig struct { - Name string - Network string - IP string -} - -type Docker struct { - info Info - Container testcontainers.Container -} - -func NewDocker(ctx context.Context, cfg DockerConfig) (*Docker, error) { - c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "nylon-debug:latest", - Entrypoint: []string{"/bin/sh", "-c", "exec sleep infinity"}, - Networks: []string{cfg.Network}, - NetworkAliases: map[string][]string{cfg.Network: {cfg.Name}}, - HostConfigModifier: func(h *container.HostConfig) { - h.Privileged = true - h.CapAdd = []string{"NET_ADMIN"} - }, - EndpointSettingsModifier: func(endpoints map[string]*network.EndpointSettings) { - endpoints[cfg.Network].IPAMConfig = &network.EndpointIPAMConfig{IPv4Address: netip.MustParseAddr(cfg.IP)} - }, - }, - Started: true, - }) - if err != nil { - return nil, err - } - return &Docker{info: Info{OS: Linux, IP: cfg.IP, WorkDir: "/app/config"}, Container: c}, nil -} - -func (d *Docker) Info() Info { return d.info } - -func (d *Docker) Run(ctx context.Context, command Command) (Result, error) { - args := append([]string{"exec", d.Container.GetContainerID(), command.Path}, command.Args...) - cmd := exec.CommandContext(ctx, "docker", args...) - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr - err := cmd.Run() - result := Result{Stdout: stdout.String(), Stderr: stderr.String()} - if cmd.ProcessState != nil { - result.ExitCode = cmd.ProcessState.ExitCode() - } - return result.check(err) -} - -func (d *Docker) WriteFile(ctx context.Context, path string, data []byte) error { - if _, err := d.Run(ctx, Command{Path: "sh", Args: []string{"-c", `mkdir -p -- "$(dirname -- "$1")"`, "machine", path}}); err != nil { - return err - } - return d.Container.CopyToContainer(ctx, data, path, 0600) -} - -func (d *Docker) Start(ctx context.Context, cmd Command) (Process, error) { - dir := d.info.WorkDir - args := []string{shellQuote(cmd.Path)} - for _, a := range cmd.Args { - args = append(args, shellQuote(a)) - } - script := "#!/bin/sh\ncd " + shellQuote(dir) + "\n" - script += strings.Join(args, " ") + " >" + shellQuote(dir+"/stdout") + " 2>" + shellQuote(dir+"/stderr") + " &\n" - script += "child=$!\nprintf '%s' \"$child\" >" + shellQuote(dir+"/pid") + "\nwait \"$child\"\ncode=$?\nprintf '%s' \"$code\" >" + shellQuote(dir+"/exit") + "\n" - if err := d.WriteFile(ctx, dir+"/run.sh", []byte(script)); err != nil { - return nil, err - } - _, err := d.Run(ctx, Command{Path: "sh", Args: []string{"-c", `touch "$1/stdout" "$1/stderr" -sh "$1/run.sh" /dev/null 2>&1 & -`, "machine", dir}}) - if err != nil { - return nil, err - } - return &dockerProcess{machine: d, dir: dir}, nil -} - -func (d *Docker) Close(ctx context.Context) error { - return d.Container.Terminate(ctx) -} - -type dockerProcess struct { - machine *Docker - dir string -} - -func (p *dockerProcess) Poll(ctx context.Context) (Result, bool, error) { - r, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `code=$(cat "$1/exit" 2>/dev/null) -printf '{"stdout":"%s","stderr":"%s","code":%s}' "$(base64 -w0 "$1/stdout")" "$(base64 -w0 "$1/stderr")" "${code:-null}"`, "machine", p.dir}}) - if err != nil { - return Result{}, false, err - } - return processOutput(r.Stdout) -} - -func (p *dockerProcess) Stop(ctx context.Context) error { - _, err := p.machine.Run(ctx, Command{Path: "sh", Args: []string{"-c", `kill "$(cat "$1/pid")"`, "machine", p.dir}}) - return err -} diff --git a/e2e/machine/machine.go b/e2e/machine/machine.go deleted file mode 100644 index 7cc2fff1..00000000 --- a/e2e/machine/machine.go +++ /dev/null @@ -1,100 +0,0 @@ -// Package machine provides the machines and guest operations used by e2e tests. -// The test runner lives on Linux; a guest may be a Linux container or Windows VM. -package machine - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "time" -) - -type OS string - -const ( - Linux OS = "linux" - Windows OS = "windows" -) - -type Info struct { - OS OS - IP string // Address on the test data network, never the management NIC. - WorkDir string -} - -// Command passes arguments separately. Shell syntax is only interpreted when -// the executable is explicitly a shell. -type Command struct { - Path string - Args []string -} - -type Result struct { - Stdout string - Stderr string - ExitCode int -} - -func (r Result) check(err error) (Result, error) { - if err != nil { - return r, err - } - if r.ExitCode != 0 { - return r, fmt.Errorf("guest command exited with code %d: %s\n%s", r.ExitCode, r.Stdout, r.Stderr) - } - return r, nil -} - -type Machine interface { - Info() Info - Run(context.Context, Command) (Result, error) - WriteFile(context.Context, string, []byte) error - Start(context.Context, Command) (Process, error) - Close(context.Context) error -} - -// Process survives the command session that launched it. Output is a cumulative -// snapshot; callers can subscribe by retaining their last byte offset. -type Process interface { - Poll(context.Context) (output Result, exited bool, err error) - Stop(context.Context) error -} - -func guestPath(m Machine, name string) string { - sep := "/" - if m.Info().OS == Windows { - sep = `\` - } - return strings.TrimRight(m.Info().WorkDir, `/\`) + sep + name -} - -func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } -func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } - -func pause(ctx context.Context, d time.Duration) error { - t := time.NewTimer(d) - defer t.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - return nil - } -} - -// Both backends return base64 output in JSON so arbitrary bytes survive transport. -func processOutput(data string) (Result, bool, error) { - var snapshot struct { - Stdout, Stderr []byte - Code *int - } - if err := json.Unmarshal([]byte(data), &snapshot); err != nil { - return Result{}, false, fmt.Errorf("decode process output: %w", err) - } - result := Result{Stdout: string(snapshot.Stdout), Stderr: string(snapshot.Stderr)} - if snapshot.Code != nil { - result.ExitCode = *snapshot.Code - } - return result, snapshot.Code != nil, nil -} diff --git a/e2e/machine/operations.go b/e2e/machine/operations.go deleted file mode 100644 index 953f26ff..00000000 --- a/e2e/machine/operations.go +++ /dev/null @@ -1,111 +0,0 @@ -package machine - -import ( - "context" - "fmt" - "net/netip" - "os" - "strconv" - "strings" - "time" - - "github.com/encodeous/nylon/state" - "github.com/goccy/go-yaml" -) - -type NylonOptions struct { - // Host paths prepared by preflight for Windows; Linux uses its Docker image. - Binary string - WintunDLL string - ExtraArgs []string -} - -// RunNylon installs config and optional artifacts, launches Nylon, and waits for -// initialization. Routing convergence is left to the scenario's assertions. -func RunNylon(ctx context.Context, m Machine, central state.CentralCfg, local state.LocalCfg, opts NylonOptions) (_ Process, err error) { - // Bound setup as well as initialization: an interrupted WinRM upload must - // not hang a test until go test's hard timeout (which bypasses cleanup). - ctx, cancelSetup := context.WithTimeout(ctx, 5*time.Minute) - defer cancelSetup() - centralData, err := yaml.Marshal(central) - if err != nil { - return nil, err - } - localData, err := yaml.Marshal(local) - if err != nil { - return nil, err - } - for name, data := range map[string][]byte{"central.yaml": centralData, "node.yaml": localData} { - if err := m.WriteFile(ctx, guestPath(m, name), data); err != nil { - return nil, fmt.Errorf("write %s: %w", name, err) - } - } - binary := "/usr/local/bin/nylon" - if m.Info().OS == Windows { - binary = guestPath(m, "nylon.exe") - for name, source := range map[string]string{"nylon.exe": opts.Binary, "wintun.dll": opts.WintunDLL} { - if err := upload(ctx, m, source, guestPath(m, name)); err != nil { - return nil, err - } - } - } - args := []string{"run", "-v", "-w", "--dbg-trace-tc", "--config", guestPath(m, "central.yaml"), "--node", guestPath(m, "node.yaml")} - args = append(args, opts.ExtraArgs...) - p, err := m.Start(ctx, Command{Path: binary, Args: args}) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - _ = p.Stop(cleanupCtx) - } - }() - var output Result - for { - var exited bool - output, exited, err = p.Poll(ctx) - if err != nil { - return nil, fmt.Errorf("read Nylon startup output: %w", err) - } - if exited { - return nil, fmt.Errorf("Nylon exited during startup (%d): %s\n%s", output.ExitCode, output.Stdout, output.Stderr) - } - if strings.Contains(output.Stdout+output.Stderr, "Nylon has been initialized") { - return p, nil - } - if err := pause(ctx, 250*time.Millisecond); err != nil { - return nil, fmt.Errorf("waiting for Nylon initialization: %w\n%s\n%s", err, output.Stdout, output.Stderr) - } - } -} - -func upload(ctx context.Context, m Machine, src, dst string) error { - data, err := os.ReadFile(src) - if err != nil { - return err - } - if err := m.WriteFile(ctx, dst, data); err != nil { - return fmt.Errorf("upload %s: %w", dst, err) - } - return nil -} - -// Ping requires all requested replies on both platforms. -func Ping(ctx context.Context, m Machine, addr netip.Addr, count int) (Result, error) { - ctx, cancel := context.WithTimeout(ctx, time.Duration(count*3+15)*time.Second) - defer cancel() - if m.Info().OS == Windows { - script := fmt.Sprintf(`$p = New-Object System.Net.NetworkInformation.Ping -$failed = $false -for ($i = 0; $i -lt %d; $i++) { - $r = $p.Send('%s', 2000) - Write-Output ($r.Status.ToString()) - if ($r.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) { $failed = $true } -} -if ($failed) { exit 1 }`, count, addr) - return m.Run(ctx, Command{Path: "powershell.exe", Args: []string{"-NoProfile", "-NonInteractive", "-Command", script}}) - } - return m.Run(ctx, Command{Path: "ping", Args: []string{"-n", "-c", strconv.Itoa(count), "-W", "2", "-w", strconv.Itoa(count * 3), addr.String()}}) -} diff --git a/e2e/machine/qemu_linux.go b/e2e/machine/qemu_linux.go deleted file mode 100644 index b7234871..00000000 --- a/e2e/machine/qemu_linux.go +++ /dev/null @@ -1,126 +0,0 @@ -package machine - -import ( - "context" - "fmt" - "net" - "net/netip" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" -) - -type WindowsConfig struct{ IP, Bridge, Dir, Image string } - -type QEMUWindows struct { - info Info - port int - dir, tap string - cmd *exec.Cmd - log *os.File -} - -// The CI host provides Docker, KVM and Ubuntu's OVMF 4M firmware. -func NewWindows(ctx context.Context, cfg WindowsConfig) (_ *QEMUWindows, err error) { - ip := netip.MustParseAddr(cfg.IP).As4() - if err := os.MkdirAll(cfg.Dir, 0755); err != nil { - return nil, err - } - w := &QEMUWindows{dir: cfg.Dir, tap: fmt.Sprintf("ny-%x", ip), info: Info{OS: Windows, IP: cfg.IP, WorkDir: `C:\nylon-e2e`}} - defer func() { - if err != nil { - _ = w.Close(context.Background()) - } - }() - overlay := filepath.Join(cfg.Dir, "disk.qcow2") - if out, err := exec.CommandContext(ctx, "qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", cfg.Image, overlay).CombinedOutput(); err != nil { - return nil, fmt.Errorf("create overlay: %w: %s", err, out) - } - vars, err := os.ReadFile("/usr/share/OVMF/OVMF_VARS_4M.fd") - if err != nil { - return nil, err - } - varsPath := filepath.Join(cfg.Dir, "uefi-vars.fd") - if err := os.WriteFile(varsPath, vars, 0600); err != nil { - return nil, err - } - dataMAC := fmt.Sprintf("52:54:%02x:%02x:%02x:%02x", ip[0], ip[1], ip[2], ip[3]) - if err := w.hostNetwork(ctx, `ip tuntap add dev "$1" mode tap user "$3"; ip link set dev "$1" master "$2"; ip link set dev "$1" up`, w.tap, cfg.Bridge, strconv.Itoa(os.Getuid())); err != nil { - return nil, err - } - listener, err := net.Listen("tcp4", "127.0.0.1:0") - if err != nil { - return nil, err - } - w.port = listener.Addr().(*net.TCPAddr).Port - listener.Close() - w.log, err = os.Create(filepath.Join(cfg.Dir, "qemu.log")) - if err != nil { - return nil, err - } - cmd := exec.Command("qemu-system-x86_64", - "-machine", "q35", "-accel", "kvm", "-cpu", "host", "-m", "2048", "-smp", "2", - "-display", "none", "-monitor", "none", "-serial", "file:"+filepath.Join(cfg.Dir, "serial.log"), - "-drive", "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd", - "-drive", "if=pflash,format=raw,file="+varsPath, - "-drive", "if=ide,format=qcow2,file="+overlay, - "-netdev", "tap,id=data,ifname="+w.tap+",script=no,downscript=no", - "-device", "virtio-net-pci,netdev=data,mac="+dataMAC, - "-netdev", fmt.Sprintf("user,id=mgmt,restrict=on,net=192.168.252.0/24,hostfwd=tcp:127.0.0.1:%d-:5985", w.port), - "-device", "virtio-net-pci,netdev=mgmt,mac=52:54:00:12:34:56") - cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} - cmd.Stdout, cmd.Stderr = w.log, w.log - if err := cmd.Start(); err != nil { - return nil, err - } - w.cmd = cmd - bootCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - for { - if _, err := w.runPS(bootCtx, "'ready'", ""); err == nil { - break - } - if err := pause(bootCtx, time.Second); err != nil { - return nil, err - } - } - script := `$adapter = Get-NetAdapter | Where-Object { ($_.MacAddress -replace '[:-]', '') -eq ` + psQuote(strings.ReplaceAll(dataMAC, ":", "")) + ` } -Set-NetIPInterface -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -Dhcp Disabled -Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false -New-NetIPAddress -InterfaceIndex $adapter.ifIndex -IPAddress ` + psQuote(cfg.IP) + ` -PrefixLength 24 | Out-Null -Set-NetFirewallProfile -Profile Domain,Private,Public -Enabled False -[void][IO.Directory]::CreateDirectory('C:\nylon-e2e')` - if _, err := w.runPS(bootCtx, script, ""); err != nil { - return nil, err - } - return w, nil -} - -func (w *QEMUWindows) Info() Info { return w.info } - -func (w *QEMUWindows) hostNetwork(ctx context.Context, script string, args ...string) error { - argv := append([]string{"run", "--rm", "--privileged", "--network", "host", "--entrypoint", "sh", "nylon-debug:latest", "-ec", script, "machine"}, args...) - out, err := exec.CommandContext(ctx, "docker", argv...).CombinedOutput() - if err != nil { - return fmt.Errorf("host network: %w: %s", err, out) - } - return nil -} - -func (w *QEMUWindows) Close(ctx context.Context) error { - if w.cmd != nil { - _ = w.cmd.Process.Kill() - _ = w.cmd.Wait() - } - if w.log != nil { - _ = w.log.Close() - } - err := w.hostNetwork(ctx, `ip link delete dev "$1"`, w.tap) - _ = os.Remove(filepath.Join(w.dir, "disk.qcow2")) - _ = os.Remove(filepath.Join(w.dir, "uefi-vars.fd")) - return err -} diff --git a/e2e/machine/winrm.go b/e2e/machine/winrm.go deleted file mode 100644 index d63033c1..00000000 --- a/e2e/machine/winrm.go +++ /dev/null @@ -1,157 +0,0 @@ -//go:build linux - -package machine - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/base64" - "fmt" - "net" - "strconv" - "strings" - "time" - - "github.com/masterzen/winrm" - "github.com/masterzen/winrm/soap" -) - -// Each SOAP request owns its NTLM connection. WinRM sends and long-poll -// receives can run concurrently without sharing an encrypted session or pool. -// Closing the socket also interrupts library calls that do not forward ctx. -type winRMTransport struct { - ctx context.Context - endpoint *winrm.Endpoint -} - -func (t *winRMTransport) Transport(endpoint *winrm.Endpoint) error { - t.endpoint = endpoint - return nil -} - -func (t *winRMTransport) Post(client *winrm.Client, message *soap.SoapMessage) (string, error) { - address := net.JoinHostPort(t.endpoint.Host, strconv.Itoa(t.endpoint.Port)) - conn, err := (&net.Dialer{Timeout: 20 * time.Second}).DialContext(t.ctx, "tcp", address) - if err != nil { - return "", err - } - defer conn.Close() - stop := context.AfterFunc(t.ctx, func() { _ = conn.Close() }) - defer stop() - transport := winrm.NewClientNTLMWithDial(func(string, string) (net.Conn, error) { return conn, nil }) - if err := transport.Transport(t.endpoint); err != nil { - return "", err - } - return transport.Post(client, message) -} - -// runPS uses an encoded command, so scripts never pass through cmd.exe quoting. -func (w *QEMUWindows) runPS(ctx context.Context, script string, input string) (Result, error) { - script = `$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue'; -[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false) -$global:LASTEXITCODE = 0 -` + script + ` -exit $LASTEXITCODE` - ctx, cancel := context.WithCancel(ctx) - defer cancel() - params := *winrm.DefaultParameters - params.Timeout = "PT10S" - params.TransportDecorator = func() winrm.Transporter { return &winRMTransport{ctx: ctx} } - endpoint := winrm.NewEndpoint("127.0.0.1", w.port, false, false, nil, nil, nil, 20*time.Second) - client, err := winrm.NewClientWithParameters(endpoint, "Administrator", "nylon", ¶ms) - if err != nil { - return Result{}, err - } - stdout, stderr, code, err := client.RunPSWithContextWithString(ctx, script, input) - return (Result{Stdout: stdout, Stderr: stderr, ExitCode: code}).check(err) -} - -func (w *QEMUWindows) Run(ctx context.Context, cmd Command) (Result, error) { - args := []string{psQuote(cmd.Path)} - for _, arg := range cmd.Args { - args = append(args, psQuote(arg)) - } - return w.runPS(ctx, "& "+strings.Join(args, " "), "") -} - -// WriteFile sends base64 lines through one WinRM shell. This avoids a separate -// PowerShell/WinRM session for every small chunk of the Nylon executable. -func (w *QEMUWindows) WriteFile(ctx context.Context, path string, data []byte) error { - expected := fmt.Sprintf("%X", sha256.Sum256(data)) - var input bytes.Buffer - for len(data) != 0 { - n := min(len(data), 36*1024) - input.WriteString(base64.StdEncoding.EncodeToString(data[:n])) - input.WriteByte('\n') - data = data[n:] - } - script := `$path = ` + psQuote(path) + ` -$file = [IO.File]::Create($path) -while ($null -ne ($line = [Console]::ReadLine())) { - $bytes = [Convert]::FromBase64String($line) - $file.Write($bytes, 0, $bytes.Length) -} -$file.Close() -(Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash` - r, err := w.runPS(ctx, script, input.String()) - if err != nil { - return err - } - if strings.TrimSpace(r.Stdout) != expected { - return fmt.Errorf("WinRM upload checksum mismatch for %s", path) - } - return nil -} - -// A scheduled SYSTEM task detaches the process from WinRM's job object. A plain -// Start-Process inside a remote shell would be terminated when that shell closes. -// SYSTEM also gives Nylon the privileges required by its Windows UAPI listener. -func (w *QEMUWindows) Start(ctx context.Context, cmd Command) (Process, error) { - runner := `$ErrorActionPreference = 'Stop' -$p = Start-Process -FilePath ` + psQuote(cmd.Path) + ` -ArgumentList ` + psQuote(strings.Join(cmd.Args, " ")) + ` -WorkingDirectory 'C:\nylon-e2e' -RedirectStandardOutput 'C:\nylon-e2e\stdout' -RedirectStandardError 'C:\nylon-e2e\stderr' -PassThru -$handle = $p.Handle -[IO.File]::WriteAllText('C:\nylon-e2e\pid', [string]$p.Id) -$p.WaitForExit() -[IO.File]::WriteAllText('C:\nylon-e2e\exit', [string]$p.ExitCode)` - if err := w.WriteFile(ctx, `C:\nylon-e2e\run.ps1`, []byte(runner)); err != nil { - return nil, err - } - script := `[IO.File]::WriteAllText('C:\nylon-e2e\stdout', '') -[IO.File]::WriteAllText('C:\nylon-e2e\stderr', '') -$action = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\nylon-e2e\run.ps1' -$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest -Register-ScheduledTask -TaskName 'nylon-e2e' -Action $action -Principal $principal | Out-Null -Start-ScheduledTask -TaskName 'nylon-e2e'` - if _, err := w.runPS(ctx, script, ""); err != nil { - return nil, err - } - return &windowsProcess{machine: w}, nil -} - -type windowsProcess struct{ machine *QEMUWindows } - -func (p *windowsProcess) Poll(ctx context.Context) (Result, bool, error) { - script := `function Read-SharedFile($path) { - $file = [IO.File]::Open($path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) - $memory = New-Object IO.MemoryStream - $file.CopyTo($memory) - $file.Close() - [Convert]::ToBase64String($memory.ToArray()) -} -$code = $null -if (Test-Path 'C:\nylon-e2e\exit') { $code = [int][IO.File]::ReadAllText('C:\nylon-e2e\exit') } -@{stdout = (Read-SharedFile 'C:\nylon-e2e\stdout'); stderr = (Read-SharedFile 'C:\nylon-e2e\stderr'); code = $code} | ConvertTo-Json -Compress` - r, err := p.machine.runPS(ctx, script, "") - if err != nil { - return Result{}, false, err - } - return processOutput(r.Stdout) -} - -func (p *windowsProcess) Stop(ctx context.Context) error { - _, err := p.machine.runPS(ctx, `Stop-Process -Id ([int][IO.File]::ReadAllText('C:\nylon-e2e\pid')) -Force -Unregister-ScheduledTask -TaskName 'nylon-e2e' -Confirm:$false -[IO.File]::WriteAllText('C:\nylon-e2e\exit', '-1')`, "") - return err -} diff --git a/e2e/machine_test.go b/e2e/machine_test.go deleted file mode 100644 index e0dbcaf3..00000000 --- a/e2e/machine_test.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build e2e && linux - -package e2e - -import ( - "net/netip" - "os" - "testing" - "time" - - "github.com/encodeous/nylon/e2e/machine" - "github.com/encodeous/nylon/protocol" - "github.com/encodeous/nylon/state" -) - -// TestMachines checks Nylon startup, IPC, ping and process shutdown on two peers. -func TestMachines(t *testing.T) { - t.Parallel() - h := NewHarness(t) - dir := h.SetupTestDir() - first, second := state.GenerateKey(), state.GenerateKey() - firstIP, secondIP := GetIP(h.Subnet, 10), GetIP(h.Subnet, 11) - central := state.CentralCfg{ - Routers: []state.RouterCfg{ - SimpleRouter("first", first.Pubkey(), "10.0.0.1", firstIP), - SimpleRouter("second", second.Pubkey(), "10.0.0.2", secondIP), - }, - Graph: []string{"first, second"}, - Timestamp: time.Now().UnixNano(), - } - centralPath := h.WriteConfig(dir, "central.yaml", central) - specs := []NodeSpec{ - {Name: "first", IP: firstIP, CentralConfigPath: centralPath, NodeConfigPath: h.WriteConfig(dir, "first.yaml", SimpleLocal("first", first))}, - {Name: "second", IP: secondIP, CentralConfigPath: centralPath, NodeConfigPath: h.WriteConfig(dir, "second.yaml", SimpleLocal("second", second))}, - } - if os.Getenv("NYLON_E2E_MACHINE") == "mixed" { - specs[0].OS, specs[1].OS = machine.Linux, machine.Windows - } - h.StartMachines(specs...) - for _, peer := range []struct{ name, destination, nextHop string }{ - {"first", "10.0.0.2", "second"}, {"second", "10.0.0.1", "first"}, - } { - h.WaitForStatus(t, peer.name, func(status *protocol.StatusResponse) bool { - return HasSelectedRoute(status, peer.destination+"/32", peer.nextHop, peer.nextHop) - }) - result, err := h.Ping(peer.name, netip.MustParseAddr(peer.destination), 3) - if err != nil { - h.PrintLogs(peer.name) - t.Fatalf("%s ping: %v\n%s\n%s", peer.name, err, result.Stdout, result.Stderr) - } - } - for name, p := range h.processes { - if err := p.Stop(h.ctx); err != nil { - t.Fatalf("stop %s: %v", name, err) - } - deadline := time.Now().Add(15 * time.Second) - for { - _, exited, err := p.Poll(h.ctx) - if err != nil { - t.Fatalf("poll stopped %s: %v", name, err) - } - if exited { - break - } - if time.Now().After(deadline) { - t.Fatalf("%s did not stop", name) - } - time.Sleep(100 * time.Millisecond) - } - } -} diff --git a/e2e/machines.go b/e2e/machines.go deleted file mode 100644 index a371889f..00000000 --- a/e2e/machines.go +++ /dev/null @@ -1,142 +0,0 @@ -//go:build e2e && linux - -package e2e - -import ( - "errors" - "net/netip" - "os" - "path/filepath" - "sync" - "time" - - "github.com/encodeous/nylon/e2e/machine" - "github.com/encodeous/nylon/state" - "github.com/goccy/go-yaml" -) - -// StartMachines is the portable replacement for StartNodes. Existing fixtures -// may continue using StartNodes/Nodes until their OS-specific operations migrate. -func (h *Harness) StartMachines(specs ...NodeSpec) { - h.t.Helper() - needsWindows := false - for i := range specs { - spec := &specs[i] - if spec.OS == "" { - spec.OS = machine.OS(os.Getenv("NYLON_E2E_MACHINE")) - if spec.OS == "" { - spec.OS = machine.Linux - } - } - needsWindows = needsWindows || spec.OS == machine.Windows - } - - h.t.Log("Running machine preflight") - var prepared windowsPrerequisites - if needsWindows { - var err error - prepared, err = preflight(h.ctx, h.RootDir) - if err != nil { - h.t.Fatal(err) - } - } - errs := make([]error, len(specs)) - var wg sync.WaitGroup - for i, spec := range specs { - wg.Go(func() { errs[i] = h.startMachine(spec, prepared) }) - } - wg.Wait() - if err := errors.Join(errs...); err != nil { - h.t.Fatal(err) - } -} - -func (h *Harness) startMachine(spec NodeSpec, prepared windowsPrerequisites) error { - osType := spec.OS - var m machine.Machine - var opts machine.NylonOptions - opts.ExtraArgs = spec.ExtraArgs - var err error - h.t.Logf("Starting %s machine %s at %s", osType, spec.Name, spec.IP) - switch osType { - case machine.Linux: - m, err = machine.NewDocker(h.ctx, machine.DockerConfig{Name: spec.Name, Network: h.Network.Name, IP: spec.IP}) - case machine.Windows: - opts.Binary, opts.WintunDLL = prepared.binary, prepared.dll - m, err = machine.NewWindows(h.ctx, machine.WindowsConfig{ - IP: spec.IP, Bridge: "br-" + h.Network.ID[:12], Image: prepared.image, - Dir: filepath.Join(h.RootDir, "workdir", "e2e", "vms", h.t.Name(), spec.Name), - }) - } - if err != nil { - return err - } - h.t.Logf("Machine %s is ready; installing Nylon and configuration", spec.Name) - // Register immediately, so config/upload/start failures still get cleanup. - h.mu.Lock() - h.Machines[spec.Name] = m - h.mu.Unlock() - centralBytes, err := os.ReadFile(spec.CentralConfigPath) - if err != nil { - return err - } - localBytes, err := os.ReadFile(spec.NodeConfigPath) - if err != nil { - return err - } - var central state.CentralCfg - var local state.LocalCfg - if err := yaml.Unmarshal(centralBytes, ¢ral); err != nil { - return err - } - if err := yaml.Unmarshal(localBytes, &local); err != nil { - return err - } - p, err := machine.RunNylon(h.ctx, m, central, local, opts) - if err != nil { - return err - } - h.t.Logf("Nylon initialized on %s", spec.Name) - h.mu.Lock() - h.processes[spec.Name] = p - h.mu.Unlock() - h.followProcess(spec.Name, p) - return nil -} - -func (h *Harness) followProcess(name string, p machine.Process) { - h.logWG.Add(1) - go func() { - defer h.logWG.Done() - var offsets [2]int - writers := [2]managerWriter{} - for i := range writers { - writers[i] = managerWriter{node: name, source: SourceStdout, manager: h.LogManager} - } - for { - r, exited, err := p.Poll(h.ctx) - if err != nil { - return - } - for i, data := range []string{r.Stdout, r.Stderr} { - _, _ = writers[i].Write([]byte(data[offsets[i]:])) - offsets[i] = len(data) - } - if exited { - return - } - select { - case <-h.ctx.Done(): - return - case <-time.After(500 * time.Millisecond): - } - } - }() -} - -func (h *Harness) Ping(name string, addr netip.Addr, count int) (machine.Result, error) { - h.mu.Lock() - m := h.Machines[name] - h.mu.Unlock() - return machine.Ping(h.ctx, m, addr, count) -} diff --git a/e2e/preflight.go b/e2e/preflight.go deleted file mode 100644 index e26e531a..00000000 --- a/e2e/preflight.go +++ /dev/null @@ -1,129 +0,0 @@ -//go:build e2e && linux - -package e2e - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "sync" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/joho/godotenv" - "github.com/klauspost/compress/zstd" -) - -type windowsPrerequisites struct{ image, binary, dll string } - -var windowsPreflight struct { - sync.Once - prepared windowsPrerequisites - err error -} - -func preflight(ctx context.Context, root string) (windowsPrerequisites, error) { - windowsPreflight.Do(func() { windowsPreflight.prepared, windowsPreflight.err = prepareWindows(ctx, root) }) - return windowsPreflight.prepared, windowsPreflight.err -} - -func prepareWindows(ctx context.Context, root string) (windowsPrerequisites, error) { - dir := filepath.Join(root, "workdir", "e2e") - p := windowsPrerequisites{ - image: filepath.Join(dir, "images", "nylon-ci-win2025.qcow2"), - binary: filepath.Join(dir, "nylon.exe"), - dll: filepath.Join(dir, "artifacts", "wintun-0.14.1-amd64.dll"), - } - if err := os.MkdirAll(filepath.Dir(p.image), 0755); err != nil { - return p, err - } - if _, err := os.Stat(p.image); os.IsNotExist(err) { - _ = godotenv.Load(filepath.Join(root, ".env")) - if err := downloadWindowsImage(ctx, p.image); err != nil { - return p, err - } - } - cmd := exec.CommandContext(ctx, "go", "build", "-o", p.binary, ".") - cmd.Dir = root - cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=windows", "GOARCH=amd64") - if out, err := cmd.CombinedOutput(); err != nil { - return p, fmt.Errorf("build Windows Nylon: %w: %s", err, out) - } - if _, err := os.Stat(p.dll); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(p.dll), 0755); err != nil { - return p, err - } - resp, err := http.Get("https://www.wintun.net/builds/wintun-0.14.1.zip") - if err != nil { - return p, err - } - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return p, err - } - archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - return p, err - } - dll, err := archive.Open("wintun/bin/amd64/wintun.dll") - if err != nil { - return p, err - } - defer dll.Close() - data, err = io.ReadAll(dll) - if err != nil { - return p, err - } - if err := os.WriteFile(p.dll, data, 0644); err != nil { - return p, err - } - } - return p, nil -} - -func downloadWindowsImage(ctx context.Context, path string) error { - const payloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - url := os.Getenv("AWS_ENDPOINT_URL_S3") + "/nylon-ci-windows-image/nylon-ci-win2025.qcow2.zst" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - req.Header.Set("X-Amz-Content-Sha256", payloadHash) - credentials := aws.Credentials{AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY")} - if err := v4.NewSigner().SignHTTP(ctx, credentials, req, payloadHash, "s3", "auto", time.Now()); err != nil { - return err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download Windows image: HTTP %d", resp.StatusCode) - } - decoder, err := zstd.NewReader(resp.Body) - if err != nil { - return err - } - defer decoder.Close() - file, err := os.Create(path + ".partial") - if err != nil { - return err - } - defer file.Close() - if _, err := io.Copy(file, decoder); err != nil { - return err - } - if err := file.Close(); err != nil { - return err - } - return os.Rename(file.Name(), path) -} diff --git a/e2e/windows/README.md b/e2e/windows/README.md new file mode 100644 index 00000000..31b0b0b8 --- /dev/null +++ b/e2e/windows/README.md @@ -0,0 +1,31 @@ +# Windows e2e + +Run on x86_64 Ubuntu with Docker, KVM and Go installed: + +```sh +sudo apt-get install qemu-system-x86 qemu-utils ovmf python3-venv unzip curl +# The current user must have read/write access to /dev/kvm and the Docker socket. +e2e/windows/run.sh /absolute/path/to/windows.qcow2 +``` + +The script builds Nylon for Linux and Windows, downloads Wintun, and installs +pywinrm in a local virtualenv. QEMU uses `/usr/bin/qemu-system-x86_64`, +`/usr/bin/qemu-img`, and `/usr/share/OVMF/OVMF_{CODE,VARS}_4M.fd`. + +The image must support UEFI, virtio networking, and WinRM with +`Administrator` / `nylon`. Each VM uses a fresh qcow2 overlay and UEFI variables; +the supplied backing image is unchanged. Guest firewalls are disabled. + +There is one test: `win1 <-> linux <-> win2`. It checks three consecutive ping +replies in both directions between the Windows nodes, and between win1 and Linux. +The Docker network and WinRM ports are fixed; run only one instance per host. +Logs stay in `workdir/e2e/windows`; VMs, overlays, TAPs and containers are removed. + +CI builds `github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025`, +decompresses its output, and passes the qcow2 path to the same script. +The private S3 Nix cache uses the existing `nylon-ci-windows-image` R2 bucket and +`AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` secrets. +Without credentials it builds without this cache. Cache download failures fall +back to local builds; uploads on pushes to main are best effort and include only +the final image output. The authenticated bucket is trusted directly, so no Nix +signing key is needed. Vault credential provisioning can replace these secrets later. diff --git a/e2e/windows/connectivity.py b/e2e/windows/connectivity.py new file mode 100644 index 00000000..aa50ecd6 --- /dev/null +++ b/e2e/windows/connectivity.py @@ -0,0 +1,186 @@ +"""One fixed topology: win1 <-> linux <-> win2, on an Ubuntu/KVM host.""" + +import functools +import http.server +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import threading +import time +import unittest + +import winrm + + +IMAGE = Path(sys.argv.pop(1)).resolve() +WORK = Path("workdir/e2e/windows").resolve() +NETWORK = "nylon-windows-e2e" +BRIDGE = "nylon-e2e" +LINUX = "nylon-e2e-linux" + + +def run(*args, **kwargs): + return subprocess.run(args, check=True, capture_output=True, text=True, **kwargs).stdout.strip() + + +def host_network(script): + run("docker", "run", "--rm", "--privileged", "--network", "host", + "--entrypoint", "sh", "nylon-debug:latest", "-ec", script) + + +def powershell(port, script): + # Each operation owns its WinRM session and closes the HTTP connection. + session = winrm.Session(f"http://127.0.0.1:{port}/wsman", auth=("Administrator", "nylon"), + transport="ntlm", operation_timeout_sec=10, read_timeout_sec=20) + try: + result = session.run_ps("$ErrorActionPreference = 'Stop';\n" + "[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false);\n" + script) + if result.status_code: + raise RuntimeError(result.std_out.decode() + result.std_err.decode()) + return result.std_out.decode().strip() + finally: + session.protocol.transport.close_session() + + +def wait_for(description, operation, timeout): + deadline = time.monotonic() + timeout + while True: + try: + return operation() + except Exception as error: + if time.monotonic() >= deadline: + raise TimeoutError(f"Waiting for {description}: {error}") from error + time.sleep(1) + + +class WindowsConnectivity(unittest.TestCase): + def start_windows(self, index): + name, port, tap = f"win{index}", 55984 + index, f"nylon-win{index}" + directory = WORK / name + directory.mkdir(exist_ok=True) + disk, variables = directory / "disk.qcow2", directory / "vars.fd" + run("/usr/bin/qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b", str(IMAGE), str(disk)) + self.addCleanup(disk.unlink) + shutil.copyfile("/usr/share/OVMF/OVMF_VARS_4M.fd", variables) + self.addCleanup(variables.unlink) + host_network(f"ip tuntap add {tap} mode tap user {os.getuid()}; " + f"ip link set {tap} master {BRIDGE}; ip link set {tap} up") + self.addCleanup(host_network, f"ip link delete {tap}") + log = (directory / "qemu.log").open("w") + self.addCleanup(log.close) + process = subprocess.Popen([ + "/usr/bin/qemu-system-x86_64", "-machine", "q35", "-accel", "kvm", "-cpu", "host", + "-m", "2048", "-smp", "2", "-display", "none", "-monitor", "none", + "-serial", f"file:{directory / 'serial.log'}", + "-drive", "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd", + "-drive", f"if=pflash,format=raw,file={variables}", + "-drive", f"if=ide,format=qcow2,file={disk}", + "-netdev", f"tap,id=data,ifname={tap},script=no,downscript=no", + "-device", f"virtio-net-pci,netdev=data,mac=52:54:00:00:00:0{index}", + "-netdev", f"user,id=mgmt,restrict=on,net=192.168.252.0/24,hostfwd=tcp:127.0.0.1:{port}-:5985", + "-device", "virtio-net-pci,netdev=mgmt,mac=52:54:00:12:34:56", + ], stdout=log, stderr=log) + + def stop(): + process.kill() + process.wait() + + self.addCleanup(stop) + + def configure(): + if process.poll() is not None: + self.fail(f"{name} QEMU exited; see {directory / 'qemu.log'}") + return powershell(port, f""" +$adapter = Get-NetAdapter | Where-Object {{ $_.MacAddress -eq '52-54-00-00-00-0{index}' }} +Set-NetIPInterface -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -Dhcp Disabled +Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | Remove-NetIPAddress -Confirm:$false +New-NetIPAddress -InterfaceIndex $adapter.ifIndex -IPAddress '172.30.250.{8 + index * 2}' -PrefixLength 24 | Out-Null +Set-NetFirewallProfile -Profile Domain,Private,Public -Enabled False +[void][IO.Directory]::CreateDirectory('C:\\nylon-e2e') +""") + print(f"Waiting for {name} WinRM and network setup...", flush=True) + wait_for(f"{name} network setup", configure, 300) + return port + + def test_connectivity(self): + run("docker", "network", "create", "--subnet", "172.30.250.0/24", "--ip-range", "172.30.250.128/25", + "--gateway", "172.30.250.1", "-o", f"com.docker.network.bridge.name={BRIDGE}", NETWORK) + self.addCleanup(run, "docker", "network", "rm", NETWORK) + + central = {"routers": [], "graph": ["win1, linux", "linux, win2"], "timestamp": time.time_ns()} + for name, address in [("win1", "10.0.0.1"), ("linux", "10.0.0.2"), ("win2", "10.0.0.3")]: + key = run("docker", "run", "--rm", "--entrypoint", "nylon", "nylon-debug:latest", "genkey") + public = run("docker", "run", "--rm", "-i", "--entrypoint", "nylon", "nylon-debug:latest", + "pubkey", input=key + "\n") + central["routers"].append({"id": name, "pubkey": public, "addresses": [address], + "endpoints": ["172.30.250.11:57175"] if name == "linux" else []}) + local = {"id": name, "key": key, "port": 57175, "interface_name": "nylon0"} + if name != "linux": + local["log_path"] = "C:\\nylon-e2e\\nylon.log" + (WORK / f"{name}.json").write_text(json.dumps(local)) + (WORK / "central.json").write_text(json.dumps(central)) + + run("docker", "run", "-d", "--name", LINUX, "--privileged", "--network", NETWORK, + "--ip", "172.30.250.11", "-v", f"{WORK}:/app/config:ro", "nylon-debug:latest", + "--config", "central.json", "--node", "linux.json") + self.addCleanup(run, "docker", "rm", "-f", LINUX) + + handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(WORK)) + server = http.server.ThreadingHTTPServer(("172.30.250.1", 0), handler) + self.addCleanup(server.server_close) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.shutdown) + url = f"http://172.30.250.1:{server.server_port}" + + ports = {} + for index in (1, 2): + name = f"win{index}" + port = ports[name] = self.start_windows(index) + powershell(port, f""" +foreach ($file in @('nylon.exe', 'wintun.dll', 'central.json', '{name}.json')) {{ + Invoke-WebRequest -UseBasicParsing ('{url}/' + $file) -OutFile ('C:\\nylon-e2e\\' + $file) +}} +# SYSTEM provides UAPI privileges and the scheduled task survives closing WinRM. +$action = New-ScheduledTaskAction -Execute 'C:\\nylon-e2e\\nylon.exe' -WorkingDirectory 'C:\\nylon-e2e' -Argument 'run -v -w --config central.json --node {name}.json' +$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +Register-ScheduledTask -TaskName 'nylon-e2e' -Action $action -Principal $principal | Out-Null +Start-ScheduledTask -TaskName 'nylon-e2e' +""") + + def collect_logs(): + (WORK / "linux.log").write_text(run("docker", "logs", LINUX)) + for name, port in ports.items(): + try: + (WORK / name / "nylon.log").write_text(powershell(port, + "Get-Content 'C:\\nylon-e2e\\nylon.log' -Raw -Encoding UTF8")) + except Exception as error: + print(f"Reading {name} logs: {error}", flush=True) + + self.addCleanup(collect_logs) + + def ping(source, destination, count): + if source == "linux": + output = run("docker", "exec", LINUX, "ping", "-n", "-c", str(count), "-W", "2", destination) + self.assertIn(f"{count} packets transmitted, {count} received", output) + return output + return powershell(ports[source], f""" +$ping = New-Object System.Net.NetworkInformation.Ping +for ($i = 0; $i -lt {count}; $i++) {{ + $reply = $ping.Send('{destination}', 2000) + if ($reply.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) {{ throw $reply.Status }} + $reply.Status +}} +""") + + for source, destination in [("win1", "10.0.0.3"), ("win2", "10.0.0.1"), + ("win1", "10.0.0.2"), ("linux", "10.0.0.1")]: + with self.subTest(source=source, destination=destination): + wait_for(f"{source} -> {destination} convergence", lambda: ping(source, destination, 1), 120) + print(f"{source} -> {destination}: {ping(source, destination, 3)}", flush=True) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/e2e/windows/run.sh b/e2e/windows/run.sh new file mode 100755 index 00000000..00249f56 --- /dev/null +++ b/e2e/windows/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +image=$(realpath "${1:?Usage: e2e/windows/run.sh IMAGE.qcow2}") +cd "$(dirname "$0")/../.." +mkdir -p workdir/e2e/windows + +docker build --target debug -t nylon-debug:latest . +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o workdir/e2e/windows/nylon.exe . +curl -fsSL https://www.wintun.net/builds/wintun-0.14.1.zip -o workdir/e2e/windows/wintun.zip +unzip -p workdir/e2e/windows/wintun.zip wintun/bin/amd64/wintun.dll > workdir/e2e/windows/wintun.dll +python3 -m venv workdir/e2e/windows/venv +workdir/e2e/windows/venv/bin/pip install --quiet pywinrm==0.5.0 +exec workdir/e2e/windows/venv/bin/python e2e/windows/connectivity.py "$image" diff --git a/go.mod b/go.mod index 7748362f..8e9e64b0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/encodeous/nylon go 1.26.3 require ( - github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/digineo/go-ping v1.2.0 github.com/dustin/go-broadcast v0.0.0-20211018055107-71439988bd91 github.com/encodeous/metric v0.0.0-20251111175231-f339c2f7c4bd @@ -13,10 +12,7 @@ require ( github.com/goccy/go-yaml v1.19.1 github.com/google/go-cmp v0.7.0 github.com/jellydator/ttlcache/v3 v3.4.0 - github.com/joho/godotenv v1.5.1 - github.com/klauspost/compress v1.18.7 github.com/kmahyyg/go-network-compo v0.2.10 - github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59 github.com/moby/moby/api v1.54.2 github.com/moby/moby/client v0.4.1 github.com/moby/term v0.5.2 @@ -36,10 +32,7 @@ require ( dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect - github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c // indirect - github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/aws/smithy-go v1.28.7 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect @@ -57,19 +50,11 @@ require ( github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jcmturner/aescts/v2 v2.0.0 // indirect - github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/goidentity/v6 v6.0.1 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect - github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.3.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect diff --git a/go.sum b/go.sum index 9732b5de..ea771222 100644 --- a/go.sum +++ b/go.sum @@ -6,16 +6,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c h1:HyK5+m8TFO4iAmMJ7u8tyUjE9C/y4/mvFRT0nqEPfJY= -github.com/Azure/go-ntlmssp v0.1.2-0.20260824092057-1c8077778c2c/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= -github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 h1:w0E0fgc1YafGEh5cROhlROMWXiNoZqApk2PDN0M1+Ns= -github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= -github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= -github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= -github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -33,8 +25,6 @@ github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/digineo/go-logwrap v0.0.0-20181106161722-a178c58ea3f0 h1:OT/LKmj81wMymnWXaKaKBR9n1vPlu+GC0VVKaZP6kzs= @@ -70,8 +60,6 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/goccy/go-yaml v1.19.1 h1:3rG3+v8pkhRqoQ/88NYNMHYVGYztCOCIZ7UQhu7H+NE= github.com/goccy/go-yaml v1.19.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= -github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -79,31 +67,10 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= -github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= -github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= -github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= -github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= -github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= -github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= -github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= -github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= -github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kmahyyg/go-network-compo v0.2.10 h1:H5rZ59xxmaZsIucz5FGV56ww+nzR3k1RFaKEN2bYRog= @@ -116,10 +83,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 h1:2ZKn+w/BJeL43sCxI2jhPLRv73oVVOjEKZjKkflyqxg= -github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= -github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59 h1:ZvQGbOpXorIxtua/7uFpXUZ1FyarecLn9BoxqJI6miY= -github.com/masterzen/winrm v0.0.0-20261004123143-57b335310d59/go.mod h1:56p25CXdyKsQWriUDfygyVGv5OvFBDY27/yzRy7NuSs= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.3.0 h1:nos4BtzzUIqB406BgQnWGMI4qib9BZ8XUHU+ucv/n1c= @@ -142,7 +105,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= @@ -165,15 +127,8 @@ github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= @@ -182,7 +137,6 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -205,52 +159,23 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= @@ -259,8 +184,6 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= From 8d7ab265d04c193403ebbd1f31c90c0e47f14c08 Mon Sep 17 00:00:00 2001 From: Dongruixuan Li Date: Sun, 1 Nov 2026 20:21:55 -0500 Subject: [PATCH 4/4] ci: read the public R2 Nix cache with its signing key --- .github/workflows/go-test.yml | 27 +++------------------------ e2e/windows/README.md | 11 +++++------ 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index ee52da2c..a2a32414 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -92,38 +92,17 @@ jobs: - uses: cachix/install-nix-action@v31 with: extra_nix_config: | + extra-substituters = https://pub-b7e688bb852e422d84f49c2dce8e3461.r2.dev + extra-trusted-public-keys = nylon-cache-1:B87/nHkgelYp2qg2AsYQ+Hmsg99ErIbbvkFCfS7u2WI= fallback = true connect-timeout = 5 download-attempts = 1 extra-system-features = kvm - name: Build Windows image id: windows_image - env: - AWS_ENDPOINT_URL_S3: ${{ secrets.AWS_ENDPOINT_URL_S3 }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - PUBLISH_CACHE: ${{ github.event_name == 'push' }} run: | - cache_args=() - if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then - # Both the Nix daemon (reads) and nix copy (writes) need credentials. - mkdir -p "$HOME/.aws" - printf '[default]\naws_access_key_id=%s\naws_secret_access_key=%s\n' \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" > "$HOME/.aws/credentials" - chmod 600 "$HOME/.aws/credentials" - sudo mkdir -p /root/.aws - sudo install -m 600 "$HOME/.aws/credentials" /root/.aws/credentials - # This private, authenticated bucket is trusted directly. - cache="s3://nylon-ci-windows-image?endpoint=${AWS_ENDPOINT_URL_S3#https://}®ion=auto&trusted=true" - cache_args=(--option extra-substituters "$cache") - fi nix build github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025 \ - --out-link "$RUNNER_TEMP/windows-image" --fallback "${cache_args[@]}" - if [[ -n "$AWS_ACCESS_KEY_ID" && "$PUBLISH_CACHE" == true ]]; then - # The output contains an already compressed qcow2; don't recompress it. - nix copy --no-recursive --to "$cache&compression=none&multipart-upload=true" "$RUNNER_TEMP/windows-image" \ - || echo '::warning::Could not upload Windows image to the Nix cache' - fi + --out-link "$RUNNER_TEMP/windows-image" --fallback zstd -d "$RUNNER_TEMP/windows-image/nylon-ci-win2025.qcow2.zst" -o "$RUNNER_TEMP/windows.qcow2" echo "qcow2=$RUNNER_TEMP/windows.qcow2" >> "$GITHUB_OUTPUT" - name: Run Windows e2e diff --git a/e2e/windows/README.md b/e2e/windows/README.md index 31b0b0b8..f9cd2084 100644 --- a/e2e/windows/README.md +++ b/e2e/windows/README.md @@ -23,9 +23,8 @@ Logs stay in `workdir/e2e/windows`; VMs, overlays, TAPs and containers are remov CI builds `github:starcatmeow/nylon-ci-windows-image/main#nylon-ci-win2025`, decompresses its output, and passes the qcow2 path to the same script. -The private S3 Nix cache uses the existing `nylon-ci-windows-image` R2 bucket and -`AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` secrets. -Without credentials it builds without this cache. Cache download failures fall -back to local builds; uploads on pushes to main are best effort and include only -the final image output. The authenticated bucket is trusted directly, so no Nix -signing key is needed. Vault credential provisioning can replace these secrets later. +CI reads the public Nix cache at +`https://pub-b7e688bb852e422d84f49c2dce8e3461.r2.dev`, trusting the +`nylon-cache-1` public key configured in the workflow. Cache misses or download +failures fall back to local builds. No cache credentials or uploads are needed; +publish the signed final image output to the cache separately.