fix(test): reduce flakiness

e2e/harness.go

@@ -212,6 +212,23 @@ func (h *Harness) WaitForStatus(t *testing.T, nodeName string, check func(*proto
 	}
 }
 
+func (h *Harness) WaitForTCPListener(t *testing.T, nodeName string, port int) {
+	t.Helper()
+	deadline := time.Now().Add(WaitTimeout)
+	var stdout, stderr string
+	var err error
+	for {
+		stdout, stderr, err = h.Exec(nodeName, []string{"ss", "-H", "-ltn", "sport", "=", fmt.Sprintf(":%d", port)})
+		if err == nil && stdout != "" {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out waiting for TCP listener on node %s port %d: %v\nStdout: %s\nStderr: %s", nodeName, port, err, stdout, stderr)
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+}
+
 func (h *Harness) ReadStatus(nodeName string) (*protocol.StatusResponse, error) {
 	stdout, _, err := h.Exec(nodeName, []string{"nylon", "status", "-i", "nylon0", "--json"})
 	if err != nil {

e2e/healthcheck_test.go

@@ -117,6 +117,7 @@ func TestHealthcheckPing(t *testing.T) {
 	msg := "hello from node 3"
 	// listen on node 1
 	bg := h.ExecBackground("node1", []string{"nc", "-l", "8888"})
+	h.WaitForTCPListener(t, "node1", 8888)
 	// send on node 3
 	stdout, stderr, err = h.Exec("node3", []string{"bash", "-c", fmt.Sprintf("echo '%s' | nc -N 10.0.1.4 8888", msg)})
 	if err != nil {

integration/harness.go

@@ -190,52 +190,43 @@ func (v *VirtualHarness) Start() chan error {
 	if v.LogLevel == nil {
 		v.LogLevel = new(slog.LevelDebug)
 	}
-	startDelay := 0 * time.Millisecond
+	for idx := range v.Central.Routers {
+		n, err := core.NewNylon(v.Central, v.Local[idx], *v.LogLevel, "", map[string]any{
+			"vnet": vn,
+		}, state.NylonOptions{DBG_log_wireguard: true}, v.Tunables)
+		if err != nil {
+			errChan <- err
+			return errChan
+		}
+		v.Nylons[idx].Store(n)
+	}
+	v.Net.Ready()
+
+	startDelay := time.Duration(0)
 	for idx, rt := range v.Central.Routers {
 		sd := startDelay
+		n := v.Nylons[idx].Load()
 		go func() {
-			time.Sleep(sd)
-			labels := pprof.Labels("nylon node", string(rt.Id))
-			n, err := core.NewNylon(v.Central, v.Local[idx], *v.LogLevel, "", map[string]any{
-				"vnet": vn,
-			}, state.NylonOptions{DBG_log_wireguard: true}, v.Tunables)
-			if err != nil {
-				errChan <- err
-				return
+			timer := time.NewTimer(sd)
+			select {
+			case <-timer.C:
+			case <-ctx.Done():
+				if !timer.Stop() {
+					select {
+					case <-timer.C:
+					default:
+					}
+				}
 			}
-			v.Nylons[idx].Store(n)
+			labels := pprof.Labels("nylon node", string(rt.Id))
 			pprof.Do(context.Background(), labels, func(_ context.Context) {
-				cErr := n.Start()
-				if cErr != nil {
-					errChan <- cErr
-					return
+				if err := n.Start(); err != nil {
+					errChan <- err
 				}
 			})
 		}()
-		startDelay += time.Millisecond * 500 // add a tiny delay so they don't try to handshake at the exact same time
-	}
-	// wait for all routers to start
-	for {
-		started := true
-		for idx, _ := range v.Central.Routers {
-			if v.Nylons[idx].Load() == nil {
-				started = false
-				break
-			}
-		}
-		if started {
-			break
-		}
-		select {
-		case <-ctx.Done():
-			return errChan
-		case <-time.After(time.Millisecond * 50):
-		case err := <-errChan:
-			errChan <- err
-			return errChan
-		}
+		startDelay += 500 * time.Millisecond
 	}
-	v.Net.Ready()
 	return errChan
 }