diff --git a/Cargo.lock b/Cargo.lock index 90f375da..07d4361b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3670,7 +3670,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.83.1" +version = "0.83.2" dependencies = [ "serde", "serde_json", @@ -3681,7 +3681,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.83.1" +version = "0.83.2" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.83.1" +version = "0.83.2" dependencies = [ "serde", "serde_json", @@ -3710,7 +3710,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.83.1" +version = "0.83.2" dependencies = [ "blake3", "libc", @@ -3736,7 +3736,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.83.1" +version = "0.83.2" dependencies = [ "axum", "base64 0.22.1", @@ -3782,7 +3782,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.83.1" +version = "0.83.2" dependencies = [ "memchr", "serde", @@ -3791,7 +3791,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.83.1" +version = "0.83.2" dependencies = [ "chrono", "serde", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.83.1" +version = "0.83.2" dependencies = [ "arc-swap", "async-stream", @@ -3856,7 +3856,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.83.1" +version = "0.83.2" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -3874,7 +3874,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.83.1" +version = "0.83.2" dependencies = [ "libc", "serde", @@ -3887,7 +3887,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.83.1" +version = "0.83.2" dependencies = [ "serde", "serde_json", @@ -3901,7 +3901,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.83.1" +version = "0.83.2" dependencies = [ "tokio", "tracing", @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.83.1" +version = "0.83.2" dependencies = [ "age", "blake3", @@ -3934,7 +3934,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.83.1" +version = "0.83.2" dependencies = [ "libc", "rand 0.10.2", @@ -3950,7 +3950,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.83.1" +version = "0.83.2" dependencies = [ "arc-swap", "flate2", @@ -3963,14 +3963,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.83.1" +version = "0.83.2" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.83.1" +version = "0.83.2" dependencies = [ "blake3", "bytes", @@ -3990,7 +3990,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.83.1" +version = "0.83.2" dependencies = [ "age", "axum", @@ -4018,7 +4018,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.83.1" +version = "0.83.2" dependencies = [ "serde", "serde_json", @@ -4027,7 +4027,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.83.1" +version = "0.83.2" dependencies = [ "libc", "notify", @@ -4038,7 +4038,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.83.1" +version = "0.83.2" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4048,7 +4048,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.83.1" +version = "0.83.2" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 2e03a2ea..25f0aaa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.83.1" +version = "0.83.2" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index 860d221e..873618ce 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -2109,6 +2109,18 @@ impl Accept for tokio::net::UnixListener { } } +/// accept 出错之后歇多久再接。对方在接之前就重置、中止的只是那一个连接,马上接下一个;别的错 +/// (打开的文件数到顶这类)立刻再 accept 还是同样的错,会原地空转、刷满日志 —— 和网关端口 +/// (axum)一样歇 1 秒 +pub(crate) fn accept_backoff(e: &std::io::Error) -> Option { + use std::io::ErrorKind::{ConnectionAborted, ConnectionRefused, ConnectionReset}; + (!matches!( + e.kind(), + ConnectionRefused | ConnectionAborted | ConnectionReset + )) + .then_some(std::time::Duration::from_secs(1)) +} + /// accept 循环。**交接时停下**([`handover::ControlListener::pause`]):停的时候不去接, /// 监听器本身留着,恢复了接着接 —— 见 [`handover`] 的模块文档。 async fn accept_loop( @@ -2120,18 +2132,24 @@ async fn accept_loop( let mut accepting = handover.control().accepting(); loop { handover::until_accepting(&mut accepting).await; - tokio::select! { - r = listener.accept_one() => match r { - Ok(stream) => hand_off( - stream, - app.clone(), - gate.clone(), - Local(Some(handover.control().closing())), - ), - Err(e) => tracing::warn!("the control plane could not accept a connection: {e}"), - }, + let r = tokio::select! { + r = listener.accept_one() => r, // 停下的那一刻放下手里这次 accept:不然它还会再接一个 - _ = accepting.wait_for(|on| !*on) => {} + _ = accepting.wait_for(|on| !*on) => continue, + }; + match r { + Ok(stream) => hand_off( + stream, + app.clone(), + gate.clone(), + Local(Some(handover.control().closing())), + ), + Err(e) => { + tracing::warn!("the control plane could not accept a connection: {e}"); + if let Some(d) = accept_backoff(&e) { + tokio::time::sleep(d).await; + } + } } } } diff --git a/crates/tw-control/src/remote.rs b/crates/tw-control/src/remote.rs index 10fdf37b..16f5e2e2 100644 --- a/crates/tw-control/src/remote.rs +++ b/crates/tw-control/src/remote.rs @@ -519,6 +519,9 @@ async fn accept_loop( Ok(x) => x, Err(e) => { tracing::warn!("the remote control port could not accept a connection: {e}"); + if let Some(d) = crate::accept_backoff(&e) { + tokio::time::sleep(d).await; + } continue; } }, @@ -582,6 +585,23 @@ mod tests { assert!(!allowed(&[], ip("10.1.2.3")), "空名单谁都不放"); } + #[test] + fn accept_errors_pause_unless_only_that_connection_went_away() { + use std::io::{Error, ErrorKind}; + for kind in [ + ErrorKind::ConnectionRefused, + ErrorKind::ConnectionAborted, + ErrorKind::ConnectionReset, + ] { + assert_eq!(crate::accept_backoff(&Error::from(kind)), None, "{kind:?}"); + } + // 打开的文件数到顶(EMFILE)报不出专门的 ErrorKind + assert_eq!( + crate::accept_backoff(&Error::other("too many open files")), + Some(Duration::from_secs(1)) + ); + } + #[test] fn five_failures_in_a_minute_bench_a_source_for_a_minute() { let r = Remote::default(); diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index b16e3d81..f3790f88 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -742,7 +742,7 @@ impl Sync { /// 同步一轮(顺带处理几处冲突),更新状态、发事件 pub async fn round(&self, decisions: &[Decision]) -> Result { let mut guard = self.run.lock().await; - // Consume this trigger before any early return, or an expired deadline spins the scheduler. + // 一开头就把到点的改动认领下来:下面几处提前返回要是把它留着,过了点的时刻会让调度原地空转 if let Ok(mut d) = self.dirty.lock() { *d = None; } @@ -1777,7 +1777,7 @@ mod tests { sync.paused .store(expected == tw_api::SyncState::Paused, Ordering::Relaxed); - // An expired dirty deadline must not override the next periodic poll forever. + // 过了点的改动时刻不能一直压过下一次定时检查 for _ in 0..2 { sync.touch(); assert!(sync.dirty.lock().unwrap().is_some()); diff --git a/release-notes/0.83.2.md b/release-notes/0.83.2.md new file mode 100644 index 00000000..a06e8a19 --- /dev/null +++ b/release-notes/0.83.2.md @@ -0,0 +1,11 @@ +This release fixes core using a full CPU core while idle after a configuration change, and makes the control plane pause instead of spinning when it cannot accept connections. + +**Upgrade notes** + +- The control-plane protocol version stays at 54; ThinkWatch Lite releases that include 0.83.0 or 0.83.1 connect to 0.83.2 and the other way round. +- The request store's schema and the configuration are unchanged; nothing is cleared. +- No new message codes. + +**No more busy loop after a configuration change.** Since 0.78.0, any local configuration change (renaming a client, adding an upstream, editing the file) scheduled a sync round 10 seconds later. When sync was off, could not be set up from its settings, or was paused in safe mode, that round ended without consuming the scheduled time, so the sync scheduler woke immediately again and kept one CPU core busy until core restarted. The scheduled time is now consumed at the start of every round. Periodic checks and retries are unchanged. + +**The control plane pauses after an accept error.** When the local control socket or the remote control port could not accept a connection for a reason other than the connection itself going away (for example when core has run out of file descriptors), it retried at once, used a full CPU core and wrote a warning on every attempt. It now waits one second before accepting again, as the gateway ports already do.