From 8b83a906fcbd94804f7df16956374496533662db Mon Sep 17 00:00:00 2001 From: Views2k <292889387+Views2k@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:03:10 -0600 Subject: [PATCH] Close EE on every Quit and keep reconnecting after a switch restart Quit kept EE open when it could not confirm that the switch lighting was returned, for example while the switch restarted, and a second Quit waited for four more gateway attempts. Every Quit now hides the window at once and makes one close attempt. When that is not confirmed, the desktop sends "exit" to the gateway, which closes private and LAN access, tells the engine to quit and exits within 10 seconds; the unconfirmed exit is recorded so the next start recovers the lighting. The engine stopped reconnecting after five failures, about a minute, and a connection lost during a write waited for Reconnect switch. Connection errors now retry for as long as the engine runs, with the delay capped at a minute. Output that a lost connection left unconfirmed is recovered with the same steps as Reconnect switch once the switch answers, and lighting resumes only if it was running. For 10 minutes after a lost connection, a switch that answers before its lighting is ready is retried too. A rejected login, a changed SSH key or a changed ownership record still stops the attempts. --- CHANGELOG.md | 5 + .../Checks/ShutdownChecks.cs | 44 ++-- desktop/EtherlightingEvolved.Native/App.cs | 46 ++--- .../ServiceHost.cs | 25 +-- docs/API.md | 11 +- docs/INTEGRATION.md | 2 +- preview/parent-lifetime.mjs | 11 +- preview/server.mjs | 15 +- src/etherlighting_evolved/engine.py | 185 ++++++++++++++--- src/etherlighting_evolved/switch.py | 10 +- tests/desktop-parent.test.mjs | 18 ++ tests/test_automatic_recovery.py | 189 ++++++++++++++++++ tests/test_connection_recovery.py | 5 +- tests/test_native_engine.py | 112 ++++++++++- tests/test_safety_remediation.py | 101 +++++++++- 15 files changed, 660 insertions(+), 119 deletions(-) create mode 100644 tests/test_automatic_recovery.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 56501d0..4703df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- EE reconnects after a switch restart on its own. It used to stop after five attempts within about a minute, before most switches finish restarting, and it waited for Reconnect switch when the connection dropped during an animation. It now retries for as long as it runs, with delays that grow to one minute. When the connection dropped before EE could confirm the switch lighting, EE checks and recovers that lighting itself once the switch answers, and resumes it only if it was running. For 10 minutes after a lost connection, a switch that answers before its lighting is ready is retried as well. A rejected login or SSH key still stops the attempts and asks you to reconnect. +- Quit closes EE even when it cannot confirm that the switch lighting was returned, for example while the switch restarts. The window closes at once, the tray icon goes away when the service has stopped, and the next start recovers the lighting. Before, EE stayed open and asked you to choose Quit again, and the second Quit took up to 25 more seconds. + ## 0.6.0 This release removes API routes, saves schedules and port names in a format that 0.5.1 cannot read, keeps the fan mode for each switch in a new place, and moves the build to Python 3.14, Node.js 24.18.0 and .NET 10. diff --git a/desktop/EtherlightingEvolved.Native.Check/Checks/ShutdownChecks.cs b/desktop/EtherlightingEvolved.Native.Check/Checks/ShutdownChecks.cs index 928207e..a6d8b36 100644 --- a/desktop/EtherlightingEvolved.Native.Check/Checks/ShutdownChecks.cs +++ b/desktop/EtherlightingEvolved.Native.Check/Checks/ShutdownChecks.cs @@ -7,8 +7,8 @@ namespace EtherlightingEvolved.Native.Check.Checks; internal static class ShutdownChecks { // A loopback fixture with its own child. It records each request, noting whether it carried the administration - // key it was started with, and leaves the recovery journal alone. When its input closes it exits cleanly only if - // told to, standing in for the gateway's own cleanup after a desktop exit. + // key it was started with, and leaves the recovery journal alone. It records the lines it reads, and on "exit" + // ends its child and exits only if told to, standing in for the gateway closing without another release. const string Gateway = """ import http from 'node:http'; import fs from 'node:fs'; @@ -18,7 +18,10 @@ internal static class ShutdownChecks fs.writeFileSync('key', key); const worker = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {stdio:'ignore', windowsHide:true}); fs.writeFileSync('child.pid', String(worker.pid)); - process.stdin.on('end', () => { if (fs.existsSync('exit-on-input-close')) { worker.kill(); process.exit(0); } }); + process.stdin.on('data', chunk => { + fs.appendFileSync('input', String(chunk)); + if (String(chunk).includes('exit') && fs.existsSync('exit-on-request')) { worker.kill(); process.exit(1); } + }); process.stdin.resume(); const server = http.createServer(async (req, res) => { const reply = (code, body) => {res.writeHead(code); res.end(JSON.stringify(body));}; @@ -48,13 +51,13 @@ public static async Task Run(string output, Action check) { await UnexpectedExit(output, check); var keys = new List(); - foreach (var scenario in new[] { "confirmed", "refused-then-cleaned", "refused-unfinished" }) + foreach (var scenario in new[] { "confirmed", "refused-then-exited", "refused-unresponsive" }) { var refused = scenario != "confirmed"; var directory = Directory.CreateDirectory(Path.Combine(output, "quit-" + Guid.NewGuid().ToString("N"))).FullName; var script = Path.Combine(directory, "gateway.mjs"); await File.WriteAllTextAsync(script, Gateway); - if (scenario == "refused-then-cleaned") await File.WriteAllTextAsync(Path.Combine(directory, "exit-on-input-close"), ""); + if (scenario == "refused-then-exited") await File.WriteAllTextAsync(Path.Combine(directory, "exit-on-request"), ""); var journal = Path.Combine(directory, "recovery.json"); await File.WriteAllTextAsync(journal, "{\"pending\":true}"); var receipt = Path.Combine(directory, "last-session.json"); @@ -69,34 +72,35 @@ public static async Task Run(string output, Action check) check(key == host.AdminKey && key.Length == 64 && key.All(c => c is >= '0' and <= '9' or >= 'a' and <= 'f'), $"{scenario}: the service receives this launch's administration key"); using var child = Process.GetProcessById(int.Parse(await File.ReadAllTextAsync(Path.Combine(directory, "child.pid")))); - var tray = host.StopAsync(force: false); + var tray = host.StopAsync(); var requests = Path.Combine(directory, "requests"); var deadline = Stopwatch.StartNew(); while (!File.Exists(requests) && deadline.Elapsed < TimeSpan.FromSeconds(5)) await Task.Delay(20); - var ipc = host.StopAsync(force: false); - var os = host.StopAsync(force: false); + var ipc = host.StopAsync(); + var os = host.StopAsync(); check(!tray.IsCompleted && !ipc.IsCompleted && !os.IsCompleted, $"{scenario}: concurrent close requests wait for the first one"); await File.WriteAllTextAsync(Path.Combine(directory, "reply"), refused ? "refused" : "accepted"); - var results = await Task.WhenAll(tray, ipc, os).WaitAsync(TimeSpan.FromSeconds(10)); + var ending = Stopwatch.StartNew(); + var results = await Task.WhenAll(tray, ipc, os).WaitAsync(TimeSpan.FromSeconds(30)); check(results.All(result => result == !refused), $"{scenario}: every close request gets the same result"); check((await File.ReadAllLinesAsync(requests)).SequenceEqual(["POST /api/workspace/close key"]), $"{scenario}: one workspace close, with the administration key, is sent for concurrent requests"); + check(!host.Running, $"{scenario}: Quit stops the service"); + var input = Path.Combine(directory, "input"); if (!refused) { - check(!host.Running && !File.Exists(receipt), "a confirmed Quit stops the service and records nothing"); + check(!File.Exists(receipt) && !File.Exists(input), "a confirmed Quit sends no exit request and records nothing"); } else { - check(host.Running && !File.Exists(receipt), $"{scenario}: an unconfirmed Quit from the tray keeps the service running"); - var forced = await host.StopAsync(force: true).WaitAsync(TimeSpan.FromSeconds(60)); - var cleaned = scenario == "refused-then-cleaned"; - check(forced == cleaned && !host.Running, cleaned - ? "Quit again lets the service finish its own cleanup and exit" - : "Quit again ends a service whose cleanup did not finish"); - check(cleaned - ? !File.Exists(receipt) - : JsonNode.Parse(await File.ReadAllTextAsync(receipt))?["outcome"]?.GetValue() == "cleanup_unconfirmed", - $"{scenario}: the unconfirmed exit is recorded only when the service had to be ended"); + check(File.Exists(input) && File.ReadAllText(input).Contains("exit"), + $"{scenario}: after a refused close the service is told to exit without another attempt"); + check(scenario == "refused-then-exited" ? ending.Elapsed < TimeSpan.FromSeconds(10) : ending.Elapsed >= TimeSpan.FromSeconds(14), + scenario == "refused-then-exited" + ? "a service that exits on request is not waited on" + : "a service that ignores the exit request is ended after its time is up"); + check(JsonNode.Parse(await File.ReadAllTextAsync(receipt))?["outcome"]?.GetValue() == "cleanup_unconfirmed", + $"{scenario}: the unconfirmed exit is recorded so the next start recovers the lighting"); } await child.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)); check(await File.ReadAllTextAsync(journal) == "{\"pending\":true}", $"{scenario}: the existing recovery journal survives desktop exit"); diff --git a/desktop/EtherlightingEvolved.Native/App.cs b/desktop/EtherlightingEvolved.Native/App.cs index 8602b32..8291aba 100644 --- a/desktop/EtherlightingEvolved.Native/App.cs +++ b/desktop/EtherlightingEvolved.Native/App.cs @@ -96,7 +96,7 @@ sealed partial class AppHost(Application app, IClassicDesktopStyleApplicationLif TrayIcon? tray; NativeMenu? menu; NativeMenuItem? browserItem; - bool quitting, forceArmed; + bool quitting; Task? quitTask; public void Start() @@ -273,7 +273,7 @@ void CreateTray() } menu.Add(new NativeMenuItemSeparator()); var quitItem = new NativeMenuItem("Quit"); - quitItem.Click += (_, _) => _ = QuitAsync(force: false); + quitItem.Click += (_, _) => _ = QuitAsync(); menu.Add(quitItem); WindowIcon? icon = null; try { icon = new WindowIcon(AssetLoader.Open(new Uri("avares://EtherlightingEvolved/Assets/ee.ico"))); } @@ -333,16 +333,15 @@ async Task HandleAsync(string command) ShowWindow(); return "ok"; } - // The installer must be able to replace the files, so its request ends EE even if cleanup is unconfirmed. - await QuitAsync(force: true); + await QuitAsync(); return service?.Running == true ? "service_still_running" : "closed"; } /// - /// Quits after returning the switch lighting. A Quit from the tray whose return is not confirmed keeps EE - /// running and says so; choosing Quit again, the installer and Windows sign-out end EE anyway. + /// Quits after trying to return the switch lighting. EE closes even when the return is not confirmed, for + /// example while the switch is restarting; the next start recovers the lighting. /// - Task QuitAsync(bool force) => quitTask ??= CloseDesktopAsync(force || forceArmed); + Task QuitAsync() => quitTask ??= CloseDesktopAsync(); void PrepareToQuit() { @@ -355,38 +354,23 @@ void PrepareToQuit() } } - async Task CloseDesktopAsync(bool force) + async Task CloseDesktopAsync() { var code = 0; - try { PrepareToQuit(); window?.Shell.Status("Returning the switch lighting…"); } + // The window goes away at once; the tray icon stays until the service has stopped. + try + { + PrepareToQuit(); + window?.Hide(); + if (tray is not null) tray.ToolTipText = "Etherlighting Evolved is closing"; + } catch (Exception) { code = 1; } var confirmed = true; - try { confirmed = service is null || await service.StopAsync(force); } + try { confirmed = service is null || await service.StopAsync(); } catch (Exception) { confirmed = false; } - if (!confirmed && !force) - { - KeepRunning(); - return; - } await Finish(confirmed && code == 0 ? 0 : 1); } - void KeepRunning() - { - forceArmed = true; - quitting = false; - quitTask = null; - // After the service has stopped, reconnecting cannot succeed; the window keeps offering to start it again. - if (workspace is not null && service is { Running: true }) - { - workspace.Closing = false; - workspace.ConnectFrames(); - } - ShowWindow(); - window?.Shell.Notify("EE could not confirm that the switch lighting was returned. Choose Quit again to close anyway; " + - "EE recovers the lighting the next time it connects.", true); - } - async Task Finish(int code) { try { window?.Shell.Studio.SaveUiState(window.Shell.PageName); } diff --git a/desktop/EtherlightingEvolved.Native/ServiceHost.cs b/desktop/EtherlightingEvolved.Native/ServiceHost.cs index d6767a1..3c33123 100644 --- a/desktop/EtherlightingEvolved.Native/ServiceHost.cs +++ b/desktop/EtherlightingEvolved.Native/ServiceHost.cs @@ -167,22 +167,14 @@ static async Task Drain(Process ended) /// /// Asks the gateway to close the workspace, which returns the switch lighting, and waits for the service to /// exit. True only when that is confirmed. Concurrent callers share one close request. - /// Without an unconfirmed close leaves the service running, so the user can try - /// again. With it (a second Quit, the installer, Windows sign-out) the gateway first gets its own cleanup - /// attempts after its input closes, and is ended only if it still has not exited; the unconfirmed exit is then - /// recorded so the next start recovers the lighting. + /// When the close is not confirmed, for example because the switch is unreachable, the service is ended + /// anyway and the unconfirmed exit is recorded, so the next start recovers the lighting. /// - public async Task StopAsync(bool force = true) + public async Task StopAsync() { Task close; lock (stopLock) close = stopping ??= Task.Run(CloseAsync); if (await close.ConfigureAwait(false)) return true; - if (!force) - { - // Let a later request ask the gateway again. - lock (stopLock) if (stopping == close) stopping = null; - return false; - } Task ending; lock (stopLock) ending = forcing ??= Task.Run(EndAsync); return await ending.ConfigureAwait(false); @@ -243,14 +235,17 @@ async Task EndAsync() { var running = process; if (running is null) return true; - // Closing the gateway's input starts its own cleanup: up to four attempts five seconds apart. - try { running.StandardInput.Close(); } catch (Exception) { } + // "exit" tells the gateway that this close was already tried. It makes no further attempts: it closes + // private and LAN access, tells the engine to quit and exits within ten seconds. Closing its input without + // "exit" would start the cleanup it runs after a desktop crash, four attempts five seconds apart. try { - await running.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(25)).ConfigureAwait(false); - if (running.ExitCode == 0) return true; + await running.StandardInput.WriteLineAsync("exit").ConfigureAwait(false); + running.StandardInput.Close(); } catch (Exception) { } + try { await running.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(15)).ConfigureAwait(false); } + catch (Exception) { } Kill(); if (Address is not null) Locations.RecordUnconfirmedExit(dataDirectory); return false; diff --git a/docs/API.md b/docs/API.md index 2079691..9403311 100644 --- a/docs/API.md +++ b/docs/API.md @@ -763,7 +763,13 @@ removing the rule. ``` `connected` can become false while logged in, for example when the switch reboots. EE reconnects automatically, -and `error` holds the last connection error until it does. +and `error` holds the last connection error until it does. Connection errors are retried for as long as EE runs, with +delays that grow to one minute. For 10 minutes after a lost connection, a switch that answers but does not yet offer +its lighting is treated as still starting and retried too. When the connection was lost before EE could confirm its +lighting, EE runs the same checks as [`POST /api/session/recover`](#post-apisessionrecover) on its own when the switch +answers, and resumes the lighting only if it was running before. A rejected login or SSH key, a changed lighting +owner, or capabilities that are still missing after 10 minutes stop the automatic attempts and set +`reconnect_required`. `pending_scene_choice` is set while a new connection waits for the user to choose its lighting (see [`POST /api/session/scene-choice`](#post-apisessionscene-choice)), otherwise `null`. `handoff_state` is `none`, @@ -1272,7 +1278,8 @@ automatic requests until an explicit user action. Normal successful topology ref `switch_health` holds the switch's memory, temperature and storage readings, `health_sample_age_ms` their age and `health_error` is `null`, `unavailable` or `invalid_response`. - `reconnect_required` is true when an interrupted connection needs - [`POST /api/session/recover`](#post-apisessionrecover) before lighting resumes. + [`POST /api/session/recover`](#post-apisessionrecover) before lighting resumes. It stays false while EE retries + the connection and recovers the lighting on its own. - `hardware_profile` describes the detected model and its physical port layout, or `null`. - The state also carries `schedule_clock` ([WebSocket](#websocket-apiws)), `identify` ([Port identification](#port-identification)), `telemetry` and `poe_summary` diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 2068230..b31cc69 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -61,7 +61,7 @@ Local processes can use the workspace and engine routes through `127.0.0.1:4180` Devices on other computers reach the workspace through the Tailscale gateway or the LAN gateway, which forward to the local gateway. Both are off until the owner turns them on and run only in live mode, where they start with the gateway, before any switch is connected. Each forwards only the routes marked remote in `preview/remote-routes.mjs`: the list is default deny, the query is ignored and paths are matched after decoding. Switch login, saved-login removal, SSH key changes, UniFi Network setup, the Tailscale and LAN access settings, workspace close and any unlisted route stay local for all audiences and devices. `POST /api/session/host-key/replace` is local in two ways: the gateways refuse it, and the engine refuses a caller whose address is not a loopback address. `POST /api/session/forget-host-key` no longer exists. A client offers Accept new key only when `GET /api/private-access` returns `local: true`, which the local gateway sends with or without administration access and the remote gateways never send. `GET /api/session` reports `ssh_host_key` and `ssh_host_key_type` for the connected switch. -Tailscale is optional. While private access is off, the gateway runs no Tailscale command until someone turns private access on, changes its port or starts the Tailscale sign-in. The one exception is a route from an earlier setting that EE has not yet seen removed: the gateway reads `tailscale serve status` at startup, on refresh and when it stops until a startup check or refresh sees that route gone and records that in `private-access.json`. At start, on each scheduled retry and on refresh or sign-in, EE adds its own Tailscale Serve route only on a port with no Serve configuration, and it never changes other Serve configuration. When an enabled private address cannot be restored at startup because Tailscale is not connected yet, the gateway repeats the startup check with increasing delays (5 seconds up to 5 minutes) until the address is ready or the setting is turned off. When the gateway stops (Quit, the installer's `--quit`, Close web session), it removes exactly EE's route within 4 seconds after the lighting is returned and keeps the setting. An unconfirmed removal is logged as `private_route_cleanup_unconfirmed` and handled at the next start or at uninstall. +Tailscale is optional. While private access is off, the gateway runs no Tailscale command until someone turns private access on, changes its port or starts the Tailscale sign-in. The one exception is a route from an earlier setting that EE has not yet seen removed: the gateway reads `tailscale serve status` at startup, on refresh and when it stops until a startup check or refresh sees that route gone and records that in `private-access.json`. At start, on each scheduled retry and on refresh or sign-in, EE adds its own Tailscale Serve route only on a port with no Serve configuration, and it never changes other Serve configuration. When an enabled private address cannot be restored at startup because Tailscale is not connected yet, the gateway repeats the startup check with increasing delays (5 seconds up to 5 minutes) until the address is ready or the setting is turned off. When the gateway stops (Quit, the installer's `--quit`, Close web session), it removes exactly EE's route within 4 seconds after the lighting is returned, or after a Quit that could not confirm the return, and keeps the setting. An unconfirmed removal is logged as `private_route_cleanup_unconfirmed` and handled at the next start or at uninstall. The private access setting has an `audience`: `owner` (default) admits only the Tailscale account that set it up, and `accounts` admits any account identity that Tailscale Serve reports for a tailnet request. `PUT /api/private-access` accepts `audience` with `enabled`; the status reports it. A tagged client can also connect when the workspace owner approves that individual device on the Windows host (see the [desktop guide](../desktop/README.md#connect-a-tagged-tailscale-device)). The host binds its approval to the Tailscale identities of both the workspace host and the selected device, and verifies the incoming device address with Tailscale WhoIs. Each approval is independent. Revoking one device leaves the others and the chosen account audience in place. The gateway refuses Funnel requests. diff --git a/preview/parent-lifetime.mjs b/preview/parent-lifetime.mjs index 9c86b99..ca498cd 100644 --- a/preview/parent-lifetime.mjs +++ b/preview/parent-lifetime.mjs @@ -1,4 +1,6 @@ -export function watchDesktopParent({input, close, report = () => {}, exhausted = () => {}, +// "exit" comes from a desktop that is quitting after its own close was not confirmed. That attempt was made, so the +// gateway calls exit instead of retrying, and the input closing afterward starts nothing. +export function watchDesktopParent({input, close, report = () => {}, exhausted = () => {}, exit = () => {}, setTimer = setTimeout, maxAttempts = 4, retryMs = 5000}) { if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > 10) throw new RangeError('Choose 1–10 cleanup attempts.'); let task, done = false, attempts = 0; @@ -14,7 +16,12 @@ export function watchDesktopParent({input, close, report = () => {}, exhausted = }); }; input.on('end', stop); - input.on('data', chunk => { if (String(chunk).includes('quit')) stop(); }); + input.on('data', chunk => { + const text = String(chunk); + if (/\bexit\b/.test(text)) { + if (!done) { done = true; exit(); } + } else if (text.includes('quit')) stop(); + }); input.resume(); return stop; } diff --git a/preview/server.mjs b/preview/server.mjs index 780d70a..a26ce0d 100644 --- a/preview/server.mjs +++ b/preview/server.mjs @@ -66,6 +66,14 @@ export function createWorkspaceServer({root,port,live,backendPort,token,child,di if(live&&result.route==='unconfirmed')diagnostics.record('private_route_cleanup_unconfirmed'); } function stop(){workspaceClosing=true;void closePrivate().catch(()=>{});server.close();child.stdin.end('quit\n');} + // For a desktop that quits after a close that was not confirmed: no further release is attempted. The engine is told + // to quit, private and LAN access close, and the engine's ownership record lets the next start recover the lighting. + async function abandon(){ + workspaceClosing=true;privateRetry.stop();server.close(); + const engineGone=child.exitCode!==null||child.signalCode!=null?Promise.resolve():new Promise(resolve=>child.once('exit',resolve)); + try{if(!child.stdin.writableEnded)child.stdin.end('quit\n');}catch{} + await Promise.allSettled([closePrivate(),engineGone]); + } // The engine never receives the administration key or the browser's cookies. function engineHeaders(req){ const headers={...req.headers,host:`127.0.0.1:${backendPort}`,authorization:`Bearer ${token}`}; @@ -220,7 +228,7 @@ export function createWorkspaceServer({root,port,live,backendPort,token,child,di }); upstream.on('error',()=>socket.destroy()); upstream.on('response',()=>socket.destroy()); socket.once('close',()=>upstream.destroy()); upstream.end(); }); - return {server,privateRetry,shutDown,readmit,stop,engineExited,closePrivate}; + return {server,privateRetry,shutDown,readmit,stop,abandon,engineExited,closePrivate}; } // The desktop app holds the key and requests its own links. A source run has no other way to give a browser access. @@ -281,7 +289,10 @@ export async function startWorkspace() { diagnostics.record('parent_cleanup_exhausted',{attempts}); workspace.readmit(); whenWorkspaceGone(process.stdin,child,()=>finishAfter(workspace.closePrivate,10000,()=>process.exit(1))); - }}); + },exit:()=>finishAfter(workspace.abandon,10000,()=>{ + if(child.exitCode===null&&child.signalCode==null)child.kill(); + process.exit(1); + })}); for(const signal of ['SIGINT','SIGTERM'])process.on(signal,workspace.stop); server.listen(port,'127.0.0.1',()=>{ for(const line of startupLines({port,desktopHost:process.env.EE_DESKTOP_HOST,admin}))console.log(line); diff --git a/src/etherlighting_evolved/engine.py b/src/etherlighting_evolved/engine.py index a7a6d2c..92bbc61 100644 --- a/src/etherlighting_evolved/engine.py +++ b/src/etherlighting_evolved/engine.py @@ -41,12 +41,13 @@ from .identify import Identification from .links import PortLink, TrafficMeter from .mailbox import Mailbox +from .ownership import OutputOwnershipError from .poe import PoePort, summarize_poe from .port_output import display_rgb, to_output from .rules import DEFAULT_RULES, FIELDS, PortFacts, Rule, first_match from .sfp import SfpLight, SfpSync from .sfp_indicators import SfpIndicators -from .switch import EtherlightSwitch, PortOutput +from .switch import EtherlightSwitch, LightingCapabilityError, PortOutput from .transport import ( AuthenticationError, HostKeyMismatchError, @@ -61,6 +62,13 @@ # Handoff to UniFi is not offered because it would have to reach the switch through the refused key. HOST_KEY_CHANGED_ERROR = ("The switch's SSH key changed. Quit EE on the computer that runs it, open it again, and " "review the new key on the Connect page.") +# The connection was lost while a lighting change was unconfirmed. The connection loop keeps running the recovery that +# Reconnect switch runs until the switch answers. +LOST_OUTPUT_ERROR = ("The switch connection was lost before EE could confirm its lighting. EE keeps trying to " + "reconnect and will check the switch lighting when it answers.") +# A restarting switch can accept SSH before its lighting service and MCU answer, which looks like missing capabilities. +SWITCH_STARTING_ERROR = ("The switch answered, but its lighting is not ready yet. EE keeps trying for 10 minutes after " + "a lost connection, which covers a switch restart.") class NotFound(KeyError): @@ -126,7 +134,8 @@ def _slug(name: str, taken: set[str]) -> str: class Engine: MAX_INSTANT_RATE = 30.0 SFP_FRAME_RATE = 50.0 - MAX_RECONNECT_ATTEMPTS = 5 + # How long after a lost connection a switch that lacks its lighting capabilities is treated as still starting. + STARTUP_GRACE = 600.0 def __init__( self, @@ -158,6 +167,13 @@ def __init__( self._reconnect_blocked = False self._loop_failed = False self._recovering = False + # A lost connection left output unconfirmed, so the connection loop recovers it (see _recover_automatically). + self._auto_recovery = False + # Lighting was running when output was blocked, so automatic recovery resumes it. release() clears this. + self._resume_after_recovery = False + self._automatic_attempt: asyncio.Task | None = None + # When the current outage began; None while connected and before the first connection. + self._lost_at: float | None = None self.budget = CommandBudget(switch, port_count=lambda: self.num_ports, fps=fps, max_commands=max_commands, on_reduced=self._budget_reduced) self._animation_origin = self.budget.animation_time @@ -286,7 +302,13 @@ def error(self, message: str | None) -> None: self._error = message def _reconnect_required(self) -> bool: - return self._reconnect_blocked or self._loop_failed or self._native_blocked or self._output_uncertain() + # Output that a lost connection left unconfirmed is recovered by the connection loop and needs no Reconnect. + return (self._reconnect_blocked or self._loop_failed + or ((self._native_blocked or self._output_uncertain()) and not self._recovers_automatically())) + + def _recovers_automatically(self) -> bool: + """Whether the connection loop runs the Reconnect switch recovery on its own.""" + return self._auto_recovery and not (self._stopping or self._reconnect_blocked or self._loop_failed) async def stop(self) -> None: self._stopping = True @@ -747,10 +769,8 @@ async def _release_sfp_port(self, port: int) -> None: await self.switch.release_sfp(port) indicators.release_confirmed(port, revision) except (TransportError, asyncio.CancelledError) as exc: - self._native_blocked = True - self.mode = "released" + self._block_output() self.error = str(exc) or "SFP release was interrupted before confirmation" - self._output_halt.set() raise finally: indicators.end_release(port) @@ -835,6 +855,12 @@ def set_output(self, *, gamma: Any = None, transition: Any = None, max_commands: def release(self) -> None: """Halt new frames; wait_for_output_idle also ends any native animation.""" self.mode = "released" + self._resume_after_recovery = False + if self._stopping and self._automatic_attempt is not None: + # stop() calls this first. Closing does not wait for an automatic recovery attempt, which can take the + # whole SSH connect timeout while the switch is unreachable; stop() then handles the unconfirmed output + # as it did before the attempt started. + self._automatic_attempt.cancel() self.stop_identify() self._output_halt.set() self._native_release_pending |= self._native_engaged or self.switch.native_uncertain @@ -872,6 +898,8 @@ def resume(self) -> None: if self._stopping: raise TransportError("The lighting session is closing. Reconnect after shutdown completes.") if self._native_blocked or self._output_uncertain(): + if self._recovers_automatically(): + raise TransportError("Lighting can resume after EE reconnects to the switch and checks its lighting.") raise TransportError("Reconnect before resuming lighting; " "the previous animation could not be confirmed stopped") if self._reconnect_blocked or self._loop_failed: @@ -896,11 +924,22 @@ async def recover_connection(self) -> None: finally: self._recovering = False - async def _recover_connection(self) -> None: - self.release() + async def _recover_connection(self, *, automatic: bool = False) -> None: + """Recover owned output. Reconnect switch (`automatic` false) then restarts stopped loops and resumes. + + The connection loop runs the automatic recovery itself, and _recover_automatically decides whether to resume. + An automatic attempt re-raises a failure that can be retried and keeps reconnects unblocked. + """ + if automatic: + # _lost already halted output. release() would also record a scene revision on every attempt. + self.mode = "released" + self._output_halt.set() + else: + self.release() + self._auto_recovery = False + self._reconnect_blocked = True self.connected = False self._native_blocked = True - self._reconnect_blocked = True self._publish(self.state_message()) await self._write_idle.wait() await self._initialization_idle.wait() @@ -910,6 +949,9 @@ async def _recover_connection(self) -> None: try: info = await self.switch.recover_output(outputs) except BaseException as exc: + if automatic and (isinstance(exc, asyncio.CancelledError) or self._retry_allowed(exc)): + raise + self._auto_recovery = False self.error = (HOST_KEY_CHANGED_ERROR if isinstance(exc, HostKeyMismatchError) else "Lighting recovery is not confirmed. Check the connection and try Reconnect switch " "again.") @@ -918,16 +960,86 @@ async def _recover_connection(self) -> None: raise TransportError(self._error) from exc self.switch_info = info self._native_release_pending = self._native_engaged = False - self._native_blocked = self._reconnect_blocked = False + self._native_blocked = self._reconnect_blocked = self._auto_recovery = False self._reconnect_failures = 0 + self._lost_at = None self.connected, self.error = True, None self.sfp_indicators.recovered() - if not self._stopping: + if not self._stopping and not automatic: self._loop_failed = False self.resume() - # The connection loop ends once reconnects are blocked, and any loop can end on an error. + # The connection loop ends once reconnects are blocked, and any loop can end on an error. The automatic + # path runs inside the connection loop and leaves stopped loops for Reconnect switch. self._tasks = [self._spawn(task.get_name()) if task.done() else task for task in self._tasks] + async def _recover_automatically(self) -> bool: + """Run the Reconnect switch recovery for output that a lost connection left unconfirmed. + + Lighting resumes only if it was running when output was blocked and was not released since. Returns True when + the attempt failed in a way that is retried after the reconnect delay. A failure that needs the user leaves + reconnect_required and the error that Reconnect switch reports. + """ + self._recovering = True + self._automatic_attempt = asyncio.current_task() + try: + await self._recover_connection(automatic=True) + except Exception as exc: + if self._reconnect_blocked: + log.warning("automatic lighting recovery stopped: %s", exc.__cause__ or exc) + return False + self._reconnect_failures += 1 + self.error = LOST_OUTPUT_ERROR + log.info("automatic lighting recovery did not finish: %s", exc) + self._publish(self.state_message()) + return True + finally: + self._recovering = False + self._automatic_attempt = None + resume, self._resume_after_recovery = self._resume_after_recovery, False + log.info("recovered the switch lighting after a lost connection") + if resume and not self._stopping and not self._loop_failed: + self.resume() + else: + self._publish(self.state_message()) + return False + + @staticmethod + def _retryable(exc: BaseException) -> bool: + """Whether a failed connection or recovery attempt is retried without the user. + + A rejected login or SSH key, a lighting ownership or capability change, and anything that is not a connection + error need the user. + """ + return (isinstance(exc, (TransportError, OSError)) + and not isinstance(exc, (AuthenticationError, HostKeyMismatchError, OutputOwnershipError))) + + def _starting_up(self, exc: BaseException) -> bool: + """A capability check failed within STARTUP_GRACE of a lost connection. + + Right after a restart the switch can answer SSH before its lighting service runs, so the capability checks fail + although nothing changed. A rejected login or SSH key and a changed ownership record are never treated this way. + """ + return (isinstance(exc, (ValueError, LightingCapabilityError)) and self._lost_at is not None + and timing.monotonic() - self._lost_at < self.STARTUP_GRACE) + + def _retry_allowed(self, exc: BaseException) -> bool: + return self._retryable(exc) or self._starting_up(exc) + + def _retry_delay(self) -> float: + """Exponential backoff with jitter after the latest failure, capped at a minute or the configured interval.""" + ceiling = max(60.0, self.reconnect_delay) + # The bounded exponent keeps the float conversion finite however long the switch stays away. + delay = min(ceiling, self.reconnect_delay * 2 ** min(self._reconnect_failures - 1, 16)) + return min(ceiling, delay * random.uniform(1.0, 1.2)) + + def _block_output(self) -> None: + """Hold output until recovery confirms the switch lighting, and remember whether lighting was running.""" + if self.mode == "running": + self._resume_after_recovery = True + self._native_blocked = True + self.mode = "released" + self._output_halt.set() + # Scenes (presets, autosave) def _telemetry_ports(self) -> tuple[int, ...]: @@ -1353,6 +1465,8 @@ def _budget_reduced(self) -> None: def _lost(self, exc: Exception) -> None: if self.connected: log.warning("lost the switch: %s", exc) + if self._lost_at is None: + self._lost_at = timing.monotonic() self.connected = False self.stop_identify() self._telemetry_generation = None @@ -1363,13 +1477,16 @@ def _lost(self, exc: Exception) -> None: self._meter = TrafficMeter() self._output_halt.set() self._bandwidth.reset() - if self._output_uncertain(): - self._native_blocked = True - self.mode = "released" - elif self.mode == "running": - self._output_halt = asyncio.Event() + # Checked after revoking precision, which makes SFP indicators owned on this connection unconfirmed. self.switch.revoke_precision() self.error = str(exc) + if self._native_blocked or self._output_uncertain(): + # The connection loop recovers this output as Reconnect switch does, without waiting for the user. + self._block_output() + self._auto_recovery = True + self.error = LOST_OUTPUT_ERROR + elif self.mode == "running": + self._output_halt = asyncio.Event() self._switch_health = None self._health_updated_at = None self.budget.reset() @@ -1382,6 +1499,11 @@ async def _keep_connected(self) -> None: while True: if self._stopping or self._reconnect_blocked or self._loop_failed: return + if self._auto_recovery and not self._recovering: + if await self._recover_automatically(): + await asyncio.sleep(self._retry_delay()) + continue + # Output blocked while connected (a release the switch did not confirm) waits for Reconnect switch. if self._native_blocked or self._output_uncertain(): self.mode = "released" self._output_halt.set() @@ -1406,19 +1528,25 @@ async def _keep_connected(self) -> None: self.switch_info = info self.connected, self.error = True, None self._reconnect_failures = 0 + self._lost_at = None self.switch.forget() self._changed() except Exception as exc: self.connected, self.error = False, str(exc) self._reconnect_failures += 1 + # A switch that is restarting can take several minutes, so connection errors retry for as long + # as the engine runs. + retryable = self._retry_allowed(exc) + if retryable and not self._retryable(exc): + self.error = SWITCH_STARTING_ERROR + log.info("the switch lighting is not ready after a lost connection: %s", exc) if self._output_uncertain(): - self._native_blocked = True - self.mode = "released" - self._output_halt.set() - permanent = isinstance(exc, (AuthenticationError, HostKeyMismatchError)) - retryable = isinstance(exc, (TransportError, OSError)) and not permanent - self._reconnect_blocked = (self._native_blocked or not retryable - or self._reconnect_failures >= self.MAX_RECONNECT_ATTEMPTS) + self._block_output() + if retryable: + # Initialization was interrupted: the next attempts run the Reconnect switch recovery. + self._auto_recovery = True + self.error = LOST_OUTPUT_ERROR + self._reconnect_blocked = not retryable if self._reconnect_blocked: self._output_halt.set() if isinstance(exc, AuthenticationError): @@ -1426,18 +1554,13 @@ async def _keep_connected(self) -> None: elif isinstance(exc, HostKeyMismatchError): self.error = HOST_KEY_CHANGED_ERROR elif not self._native_blocked: - self.error = ( - "Automatic reconnect stopped. Reconnect your switch to try again." - if retryable else f"{exc}. Reconnect your switch after checking its settings." - ) + self.error = f"{exc}. Reconnect your switch after checking its settings." with contextlib.suppress(Exception): await self.switch.close() self._publish(self.state_message()) if self._reconnect_blocked: return - ceiling = max(60.0, self.reconnect_delay) - delay = min(ceiling, self.reconnect_delay * 2 ** (self._reconnect_failures - 1)) - await asyncio.sleep(min(ceiling, delay * random.uniform(1.0, 1.2))) + await asyncio.sleep(self._retry_delay()) continue await asyncio.sleep(1.0) diff --git a/src/etherlighting_evolved/switch.py b/src/etherlighting_evolved/switch.py index 9549b47..569e583 100644 --- a/src/etherlighting_evolved/switch.py +++ b/src/etherlighting_evolved/switch.py @@ -38,6 +38,14 @@ TransportError, ) + +class LightingCapabilityError(OutputOwnershipError): + """The switch answered without the lighting capabilities that EE's owned output was recorded with. + + A switch that is still starting answers this way too, so the engine retries it for a while after a lost connection. + """ + + WRITE_FAILED = "__EE_LED_WRITE_FAILED__" MCU_SCHEMA_EXIT = "__EE_MCU_SCHEMA_EXIT__" MCU_SCHEMA_COMMAND = ( @@ -358,7 +366,7 @@ async def recover_output(self, outputs: list[PortOutput]) -> dict: if (not info.get("port_rgb") or self.sfp_ports != old_sfp_ports or self.hardware_profile is not old_profile or self.num_ports != old_copper_count or not self.native_breathing or (sfp and not self.scoped_sfp)): - raise OutputOwnershipError("The switch lighting capabilities changed. Keep output stopped.") + raise LightingCapabilityError("The switch lighting capabilities changed. Keep output stopped.") generation = self.transport.lighting_generation if generation is None: raise OutputOwnershipError("The lighting connection could not be verified.") diff --git a/tests/desktop-parent.test.mjs b/tests/desktop-parent.test.mjs index ef31dd3..f190259 100644 --- a/tests/desktop-parent.test.mjs +++ b/tests/desktop-parent.test.mjs @@ -56,6 +56,24 @@ test('an unconfirmed close can retry rather than pretending output stopped',asyn assert.equal(closes,1);retry();await new Promise(resolve=>setImmediate(resolve));assert.equal(closes,2); }); +test('a desktop that sends exit after its own refused close gets no further close attempts',async()=>{ + const input=new PassThrough(),timers=[];let closes=0,exits=0; + watchDesktopParent({input,close:async()=>{closes++;},exit:()=>exits++,setTimer:fn=>timers.push(fn)}); + input.write('exit\r\n');input.end();await new Promise(resolve=>setImmediate(resolve)); + assert.equal(exits,1);assert.equal(closes,0);assert.equal(timers.length,0); + input.emit('data','exit\n');assert.equal(exits,1); +}); + +test('exit during a retry wait cancels the retry',async()=>{ + const input=new PassThrough(),timers=[];let closes=0,exits=0; + watchDesktopParent({input,close:async()=>{closes++;throw Error('unconfirmed');},exit:()=>exits++,setTimer:fn=>timers.push(fn)}); + input.write('quit\n');await new Promise(resolve=>setImmediate(resolve)); + assert.equal(closes,1);assert.equal(timers.length,1); + input.write('exit\n');await new Promise(resolve=>setImmediate(resolve)); + timers.shift()();await new Promise(resolve=>setImmediate(resolve)); + assert.equal(exits,1);assert.equal(closes,1); +}); + test('blocked output does not claim automatic reconnect is running',()=>{ const session={logged_in:true,connected:false,reconnect_required:true}; assert.equal(connectionLabel(session),'Switch connection needs attention'); diff --git a/tests/test_automatic_recovery.py b/tests/test_automatic_recovery.py new file mode 100644 index 0000000..c5b75f4 --- /dev/null +++ b/tests/test_automatic_recovery.py @@ -0,0 +1,189 @@ +"""Automatic recovery after a lost connection leaves lighting output unconfirmed, for example when the switch restarts +while EE writes an animation frame. These tests use the real switch recovery and output ownership records.""" + +import asyncio + +import pytest +from test_connection_recovery import PEER, interrupted +from test_native_switch import Clock, ready +from test_sfp_precision_transport import SfpPrecisionTransport + +from etherlighting_evolved import engine as engine_module +from etherlighting_evolved import timing +from etherlighting_evolved.engine import HOST_KEY_CHANGED_ERROR, LOST_OUTPUT_ERROR, Engine +from etherlighting_evolved.ownership import OutputJournal +from etherlighting_evolved.transport import HostKeyMismatchError, TransportError, UnreachableError + +RECOVERY_UNCONFIRMED = "Lighting recovery is not confirmed. Check the connection and try Reconnect switch again." + + +@pytest.fixture +def clock(monkeypatch): + clock = Clock() + monkeypatch.setattr(timing, "monotonic", lambda: clock.now) + monkeypatch.setattr(timing, "wait", clock.wait) + + +def loop_sleeps(monkeypatch, *, until, on_wait=None): + """Replace asyncio.sleep so the connection loop's waits are recorded instead of taken. + + Zero-second sleeps (command pacing) pass through. Once `until()` is true the loop ends with CancelledError; + otherwise `on_wait(count)` runs during the wait. + """ + waits, sleep = [], asyncio.sleep + + async def recorded(seconds): + if seconds == 0: + return await sleep(0) + waits.append(seconds) + if until(): + raise asyncio.CancelledError + if on_wait is not None: + on_wait(len(waits)) + await sleep(0) + + monkeypatch.setattr(engine_module.asyncio, "sleep", recorded) + monkeypatch.setattr(engine_module.random, "uniform", lambda low, high: 1.0) + return waits + + +def restarting(transport, failure): + """Make the switch fail each attempt until the returned function brings it back.""" + connect, attempts = transport.connect, [] + + async def reconnect(): + attempts.append(failure) + if failure == "unreachable" and not online: + raise UnreachableError("Can't reach fixture.invalid:22 over SSH (timed out)") + if failure == "network" and not online: + raise OSError("fixture network is unreachable") + await connect() + transport.lighting_generation = object() + + def answer(): + nonlocal online + online = True + transport.failure = None + + online = False + transport.connect = reconnect + if failure == "baseline": + transport.failure = ("behavior", "steady") + return attempts, answer + + +@pytest.mark.anyio +@pytest.mark.parametrize("failure", ["unreachable", "network", "baseline"]) +async def test_switch_restart_during_a_frame_is_recovered_and_lighting_resumes(tmp_path, clock, monkeypatch, + failure): + switch, transport = await interrupted(tmp_path) + engine = Engine(switch, rules=[], transition=0) + engine.set_effect("comet") + engine.connected = True + scene = engine.snapshot() + attempts, answer = restarting(transport, failure) + engine._lost(TransportError("the lighting socket stopped answering")) + assert engine.mode == "released" and not engine.connected + assert not engine.state()["reconnect_required"] and engine.error == LOST_OUTPUT_ERROR + revision, seen = engine.revision, [] + + def on_wait(count): + seen.append((engine.state()["reconnect_required"], engine.error, engine.mode, engine.revision)) + if count == 7: + answer() + + waits = loop_sleeps(monkeypatch, until=lambda: engine.connected, on_wait=on_wait) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + # Seven failed attempts, more than the five that used to end automatic reconnect, with the delay capped at a + # minute. Failed attempts change no scene revision. + assert waits == [3, 6, 12, 24, 48, 60, 60, 1] + assert seen == [(False, LOST_OUTPUT_ERROR, "released", revision)] * 7 + assert len(attempts) == 8 + assert engine.connected and engine.mode == "running" and engine.error is None + assert not engine.state()["reconnect_required"] and engine.snapshot() == scene + assert not switch.output_lease.uncertain and not switch.native_uncertain + + +@pytest.mark.anyio +@pytest.mark.parametrize("released", ["before_the_loss", "during_the_outage"]) +async def test_recovery_leaves_released_lighting_released(tmp_path, clock, monkeypatch, released): + switch, transport = await interrupted(tmp_path) + engine = Engine(switch, rules=[], transition=0) + engine.connected = True + _, answer = restarting(transport, "unreachable") + if released == "before_the_loss": + engine.release() + engine._lost(TransportError("the lighting socket stopped answering")) + + def on_wait(count): + if released == "during_the_outage": + engine.release() + answer() + + waits = loop_sleeps(monkeypatch, until=lambda: engine.connected, on_wait=on_wait) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == [3, 1] + assert engine.connected and engine.mode == "released" and engine.error is None + assert not engine.state()["reconnect_required"] and not switch.output_lease.uncertain + engine.resume() + assert engine.mode == "running" + + +@pytest.mark.anyio +@pytest.mark.parametrize("failure,message", [("host_key", HOST_KEY_CHANGED_ERROR), + ("ownership", RECOVERY_UNCONFIRMED)]) +async def test_failure_that_needs_the_user_stops_automatic_recovery(tmp_path, clock, monkeypatch, failure, message): + switch, transport = await interrupted(tmp_path) + engine = Engine(switch, rules=[], transition=0) + engine.connected = True + restarting(transport, "unreachable") + + async def changed_key(): + raise HostKeyMismatchError("fixture changed key") + + def on_wait(count): + # The switch answers with another SSH key, or the lighting ownership record no longer names this session. + if failure == "host_key": + transport.connect = changed_key + else: + switch.output_lease.paths[0].write_text("{}") + + engine._lost(TransportError("the lighting socket stopped answering")) + waits = loop_sleeps(monkeypatch, until=lambda: False, on_wait=on_wait) + await engine._keep_connected() + assert waits == [3] + assert engine.error == message and engine.state()["reconnect_required"] + assert not engine.connected and engine.mode == "released" + assert switch.output_lease.uncertain and all(path.exists() for path in switch.output_lease.paths) + with pytest.raises(TransportError, match="Reconnect before resuming"): + engine.resume() + await engine._keep_connected() + assert waits == [3] + + +@pytest.mark.anyio +async def test_interrupted_initialization_on_reconnect_continues_with_recovery(tmp_path, clock, monkeypatch): + transport = SfpPrecisionTransport() + transport.host_key_fingerprint = PEER + switch, _ = await ready(transport) + switch.output_lease = OutputJournal(tmp_path).claim("fixture.invalid", 22, PEER) + engine = Engine(switch, rules=[], transition=0) + engine.connected = True + engine._lost(TransportError("fixture connection reset")) + # Nothing was in flight, so this is a plain reconnect. + assert engine.mode == "running" and not engine.state()["reconnect_required"] + _, answer = restarting(transport, "baseline") + seen = [] + + def on_wait(count): + seen.append((engine.state()["reconnect_required"], engine.error, engine.mode)) + answer() + + waits = loop_sleeps(monkeypatch, until=lambda: engine.connected, on_wait=on_wait) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == [3, 1] + assert seen == [(False, LOST_OUTPUT_ERROR, "released")] + assert engine.connected and engine.mode == "running" and not switch.output_lease.uncertain diff --git a/tests/test_connection_recovery.py b/tests/test_connection_recovery.py index 3a2296f..d19c0cf 100644 --- a/tests/test_connection_recovery.py +++ b/tests/test_connection_recovery.py @@ -106,7 +106,10 @@ async def test_engine_recovery_preserves_scene_and_resumes_only_after_ack(tmp_pa engine.connected = True before = engine.snapshot() engine._lost(TransportError("cable disconnected")) - assert engine.state()["reconnect_required"] + # The connection loop would start this recovery on its own; Reconnect switch runs the same steps. + assert engine.mode == "released" and not engine.state()["reconnect_required"] + with pytest.raises(TransportError): + engine.resume() await engine.recover_connection() assert engine.connected and engine.mode == "running" assert engine.snapshot() == before diff --git a/tests/test_native_engine.py b/tests/test_native_engine.py index e758180..c3448e6 100644 --- a/tests/test_native_engine.py +++ b/tests/test_native_engine.py @@ -310,24 +310,128 @@ async def test_connection_does_not_become_ready_until_baseline_is_acknowledged() @pytest.mark.anyio -async def test_native_uncertainty_stops_automatic_reconnect(): +async def test_native_uncertainty_runs_recovery_instead_of_a_plain_reconnect(): engine = native_engine() engine.connected = True engine.switch.uncertain = True + recovering, recoveries = asyncio.Event(), [] + + async def recover_output(outputs): + recoveries.append(outputs) + recovering.set() + await asyncio.Future() + + engine.switch.recover_output = recover_output engine._lost(TransportError("unconfirmed native state")) task = asyncio.create_task(engine._keep_connected()) try: - await asyncio.sleep(0) + await asyncio.wait_for(recovering.wait(), 1) assert engine.mode == "released" and not engine.connected - assert not engine.switch.calls - with pytest.raises(TransportError): + # A plain reconnect would initialize output over the unconfirmed native state. + assert not engine.switch.calls and len(recoveries) == 1 + assert not engine.state()["reconnect_required"] + with pytest.raises(TransportError, match="after EE reconnects"): engine.resume() + with pytest.raises(TransportError, match="already in progress"): + await engine.recover_connection() finally: task.cancel() with pytest.raises(asyncio.CancelledError): await task +def loop_sleeps(monkeypatch, until): + """Replace asyncio.sleep with a recorder that ends the loop with CancelledError once `until()` is true.""" + waits, sleep = [], asyncio.sleep + + async def recorded(seconds): + waits.append(seconds) + if until(): + raise asyncio.CancelledError + await sleep(0) + + monkeypatch.setattr(asyncio, "sleep", recorded) + return waits + + +@pytest.mark.anyio +async def test_failed_release_while_connected_does_not_start_automatic_recovery(monkeypatch): + engine = native_engine() + engine.connected = True + engine._native_engaged = True + engine.switch.fail_quiesce = True + recoveries = [] + + async def recover_output(outputs): + recoveries.append(outputs) + return {} + + engine.switch.recover_output = recover_output + engine.release() + with pytest.raises(TransportError, match="not acknowledged"): + await engine.wait_for_output_idle() + waits = loop_sleeps(monkeypatch, lambda: len(waits) == 3) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == [1.0, 1.0, 1.0] and not recoveries and engine.switch.calls == ["quiesce"] + assert engine.connected and engine.mode == "released" and engine.state()["reconnect_required"] + assert engine.error == "native cleanup was not acknowledged" + with pytest.raises(TransportError, match="Reconnect before resuming"): + engine.resume() + + +@pytest.mark.anyio +async def test_release_that_failed_before_a_lost_connection_is_recovered_without_resuming(monkeypatch): + engine = native_engine() + engine.connected = True + engine._native_engaged = True + engine.switch.fail_quiesce = True + engine.release() + with pytest.raises(TransportError): + await engine.wait_for_output_idle() + recoveries = [] + + async def recover_output(outputs): + recoveries.append(outputs) + engine.switch.uncertain = False + return {"port_rgb": True} + + engine.switch.recover_output = recover_output + engine._lost(TransportError("fixture connection reset")) + assert not engine.state()["reconnect_required"] + waits = loop_sleeps(monkeypatch, lambda: engine.connected) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == [1.0] and len(recoveries) == 1 + assert engine.connected and engine.mode == "released" and engine.error is None + assert not engine.state()["reconnect_required"] + engine.resume() + assert engine.mode == "running" + + +@pytest.mark.anyio +async def test_stop_does_not_wait_for_an_automatic_recovery_attempt(): + engine = native_engine() + engine.connected = True + engine.switch.uncertain = True + entered, canceled = asyncio.Event(), asyncio.Event() + + async def unreachable(outputs): + entered.set() + try: + await asyncio.Future() + finally: + canceled.set() + + engine.switch.recover_output = unreachable + engine._lost(TransportError("fixture connection reset")) + engine._tasks = [engine._spawn("ee-connection")] + await asyncio.wait_for(entered.wait(), 1) + await asyncio.wait_for(engine.stop(), 1) + assert canceled.is_set() and engine._tasks == [] and not engine.connected + assert engine.switch.calls == ["quiesce", "close"] + + @pytest.mark.anyio async def test_release_drains_the_real_render_loop_before_native_quiescence(): engine = native_engine() diff --git a/tests/test_safety_remediation.py b/tests/test_safety_remediation.py index 5c890dd..86cff18 100644 --- a/tests/test_safety_remediation.py +++ b/tests/test_safety_remediation.py @@ -9,12 +9,14 @@ from etherlighting_evolved import api, cli from etherlighting_evolved import engine as engine_module from etherlighting_evolved.engine import Engine +from etherlighting_evolved.ownership import OutputOwnershipError from etherlighting_evolved.session import LoginError, SessionManager from etherlighting_evolved.switch import ( BOARD_LIGHTING_INFO, PORT_FIRMWARE_HASH_COMMAND, RUNNING_DAEMON_HASH, EtherlightSwitch, + LightingCapabilityError, parse_mcu_probe, ) from etherlighting_evolved.transport import ( @@ -125,7 +127,8 @@ async def close(self): @pytest.mark.anyio @pytest.mark.parametrize("error", [AuthenticationError("fixture"), HostKeyMismatchError("fixture"), - ValueError("invalid configuration"), RuntimeError("unexpected failure")]) + OutputOwnershipError("fixture"), ValueError("invalid configuration"), + RuntimeError("unexpected failure")]) async def test_permanent_reconnect_failure_stops_and_requires_manual_connection(error): switch = ReconnectSwitch([error]) engine = Engine(switch, rules=[]) @@ -156,23 +159,103 @@ async def test_removed_rgb_capability_does_not_initialize_or_retry(): assert "Per-port lighting is unavailable" in engine.error +def lost_connection(engine): + engine.connected = True + engine._lost(UnreachableError("fixture")) + + +@pytest.mark.anyio +async def test_missing_lighting_right_after_a_lost_connection_is_retried(monkeypatch): + # A restarting switch can answer SSH before its lighting service runs. + switch = ReconnectSwitch([], supported=False) + engine = Engine(switch, rules=[]) + lost_connection(engine) + waits, states = [], [] + + async def sleep(seconds): + waits.append(seconds) + if engine.connected: + raise asyncio.CancelledError + states.append((engine.state()["reconnect_required"], engine.error)) + if len(waits) == 2: + switch.supported = True + + monkeypatch.setattr(engine_module.asyncio, "sleep", sleep) + monkeypatch.setattr(engine_module.random, "uniform", lambda low, high: 1.0) + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == [3, 6, 1] + assert states == [(False, engine_module.SWITCH_STARTING_ERROR)] * 2 + assert switch.attempts == 3 and switch.initialized == 1 + assert engine.connected and engine.error is None and engine._lost_at is None + + +@pytest.mark.anyio +async def test_missing_lighting_long_after_a_lost_connection_stops(): + switch = ReconnectSwitch([], supported=False) + engine = Engine(switch, rules=[]) + lost_connection(engine) + engine._lost_at -= Engine.STARTUP_GRACE + await engine._keep_connected() + assert switch.attempts == 1 and switch.initialized == 0 + assert engine.state()["reconnect_required"] and "Per-port lighting is unavailable" in engine.error + + +def test_only_missing_lighting_is_retried_while_a_switch_starts(): + engine = Engine(ReconnectSwitch([]), rules=[]) + capability = LightingCapabilityError("fixture capabilities changed") + unavailable = ValueError("Per-port lighting is unavailable on this switch connection") + # Before any connection was lost, missing lighting is a configuration problem. + assert not engine._retry_allowed(capability) and not engine._retry_allowed(unavailable) + lost_connection(engine) + assert engine._retry_allowed(capability) and engine._retry_allowed(unavailable) + for error in (OutputOwnershipError("fixture owner changed"), AuthenticationError("fixture"), + HostKeyMismatchError("fixture"), RuntimeError("fixture")): + assert not engine._retry_allowed(error) + engine._lost_at -= Engine.STARTUP_GRACE + assert not engine._retry_allowed(capability) and not engine._retry_allowed(unavailable) + + +@pytest.mark.anyio +@pytest.mark.parametrize("error", [AuthenticationError("fixture"), HostKeyMismatchError("fixture")]) +async def test_rejected_login_right_after_a_lost_connection_still_stops(error): + switch = ReconnectSwitch([error]) + engine = Engine(switch, rules=[]) + lost_connection(engine) + await engine._keep_connected() + assert switch.attempts == 1 and engine.state()["reconnect_required"] + + @pytest.mark.anyio @pytest.mark.parametrize("error_type", [UnreachableError, TransportError, OSError]) -async def test_transient_retries_back_off_and_stop_after_five_attempts(monkeypatch, error_type): - switch = ReconnectSwitch([error_type("fixture") for _ in range(5)]) +async def test_transient_failures_keep_retrying_with_a_capped_delay(monkeypatch, error_type): + # A restarting switch can stay away longer than five attempts and their backoff. + switch = ReconnectSwitch([error_type("fixture") for _ in range(8)]) engine = Engine(switch, rules=[]) - waits = [] + waits, states = [], [] async def sleep(seconds): waits.append(seconds) + if engine.connected: + raise asyncio.CancelledError + states.append((engine.state()["reconnect_required"], engine.error, switch.initialized)) monkeypatch.setattr(engine_module.asyncio, "sleep", sleep) monkeypatch.setattr(engine_module.random, "uniform", lambda low, high: 1.1) - await engine._keep_connected() - assert switch.attempts == switch.closed == 5 - assert waits == pytest.approx([3.3, 6.6, 13.2, 26.4]) - assert engine.state()["reconnect_required"] and "Automatic reconnect stopped" in engine.error - assert switch.initialized == 0 + with pytest.raises(asyncio.CancelledError): + await engine._keep_connected() + assert waits == pytest.approx([3.3, 6.6, 13.2, 26.4, 52.8, 60, 60, 60, 1]) + assert states == [(False, "fixture", 0)] * 8 + assert switch.attempts == 9 and switch.closed == 8 and switch.initialized == 1 + assert engine.connected and engine.error is None and not engine.state()["reconnect_required"] + + +@pytest.mark.parametrize("interval,expected", [(3.0, 60.0), (300.0, 300.0)]) +def test_retry_delay_stays_finite_after_a_long_outage(monkeypatch, interval, expected): + engine = Engine(ReconnectSwitch([]), rules=[], reconnect_delay=interval) + monkeypatch.setattr(engine_module.random, "uniform", lambda low, high: high) + engine._reconnect_failures = 100_000 + assert engine._retry_delay() == expected @pytest.mark.anyio