diff --git a/crates/buzz-acp/src/base_prompt.md b/crates/buzz-acp/src/base_prompt.md index e360d24982..295938e18f 100644 --- a/crates/buzz-acp/src/base_prompt.md +++ b/crates/buzz-acp/src/base_prompt.md @@ -7,6 +7,7 @@ The `buzz` CLI is your primary interface. Auth env vars: `BUZZ_RELAY_URL`, `BUZZ | Group | Key commands | |-------|-------------| | `buzz agents` | `draft-create`, `draft-update` | +| `buzz voice` | `join`, `remove`, `mute`, `unmute`, `set-voice` | | `buzz messages` | `send`, `get`, `thread`, `search` | | `buzz channels` | `list`, `get`, `create`, `join`, `members` | | `buzz canvas` | `get`, `set` | @@ -22,6 +23,8 @@ The `buzz` CLI is your primary interface. Auth env vars: `BUZZ_RELAY_URL`, `BUZZ Run `buzz --help` or `buzz --help` for full usage. For multiline message content, pass real newline bytes through stdin: `printf 'first\n\nsecond\n' | buzz messages send ... --content -`. Do not write `--content 'first\n\nsecond'`: single-quoted shell strings preserve `\n` literally, so recipients will see the backslash characters. `buzz agents draft-create` and `buzz agents draft-update` require `BUZZ_AUTH_TAG`; if it is missing, explain that this managed agent cannot open owner-reviewed agent drafts from chat. +Voice-room controls also require `BUZZ_AUTH_TAG`. Use `buzz voice join --agent-name ` to add an existing personal agent without UI automation; run `buzz voice --help` for removal, muting, output, and voice-selection controls. These commands control the owner's currently open Voice room and do not create agents. + When opening a pull request in response to channel work, always pass `--channel ` using the UUID from `[Context]`. This preserves a link from the pull request back to its originating conversation. ## Conversational Agent Creation diff --git a/crates/buzz-cli/src/commands/agents.rs b/crates/buzz-cli/src/commands/agents.rs index 58564a45c2..3155bd3a51 100644 --- a/crates/buzz-cli/src/commands/agents.rs +++ b/crates/buzz-cli/src/commands/agents.rs @@ -16,7 +16,7 @@ pub async fn dispatch(command: AgentsCmd, client: &BuzzClient) -> Result<(), Cli display_name, system_prompt, } => { - let owner = require_owner(client)?; + let owner = require_owner(client, "agent draft requests")?; let built = build_create( client.keys(), &owner, @@ -53,7 +53,7 @@ pub async fn dispatch(command: AgentsCmd, client: &BuzzClient) -> Result<(), Cli model, respond_to, } => { - let owner = require_owner(client)?; + let owner = require_owner(client, "agent draft requests")?; let built = build_update( client.keys(), &owner, @@ -151,12 +151,14 @@ pub async fn dispatch(command: AgentsCmd, client: &BuzzClient) -> Result<(), Cli } } -/// Require `BUZZ_AUTH_TAG` and parse the owner pubkey from it. Used only by -/// the `draft-create` and `draft-update` paths. -fn require_owner(client: &BuzzClient) -> Result { +/// Require `BUZZ_AUTH_TAG` and parse the owner pubkey from it. +pub(crate) fn require_owner( + client: &BuzzClient, + request_kind: &str, +) -> Result { let hex = client .auth_tag_owner_hex() - .ok_or_else(|| CliError::Auth("agent draft requests require BUZZ_AUTH_TAG".into()))?; + .ok_or_else(|| CliError::Auth(format!("{request_kind} require BUZZ_AUTH_TAG")))?; PublicKey::parse(&hex).map_err(|e| CliError::Auth(format!("invalid owner attestation: {e}"))) } diff --git a/crates/buzz-cli/src/commands/mod.rs b/crates/buzz-cli/src/commands/mod.rs index 1ccc37a702..adbc88faa2 100644 --- a/crates/buzz-cli/src/commands/mod.rs +++ b/crates/buzz-cli/src/commands/mod.rs @@ -18,6 +18,7 @@ pub mod repos; pub mod social; pub mod upload; pub mod users; +pub mod voice; pub mod workflows; use crate::{client::normalize_write_response, error::CliError}; diff --git a/crates/buzz-cli/src/commands/voice.rs b/crates/buzz-cli/src/commands/voice.rs new file mode 100644 index 0000000000..2f4d91aaf2 --- /dev/null +++ b/crates/buzz-cli/src/commands/voice.rs @@ -0,0 +1,64 @@ +use serde_json::json; + +use crate::commands::agents::require_owner; +use crate::error::CliError; +use crate::voice_management::{build, VoiceAgentRef, VoiceRoomCommand}; +use crate::{client::BuzzClient, VoiceCmd}; + +pub async fn dispatch(command: VoiceCmd, client: &BuzzClient) -> Result<(), CliError> { + let (action, command) = match command { + VoiceCmd::Join(agent) => ( + "join", + VoiceRoomCommand::Join(VoiceAgentRef::try_from(agent)?), + ), + VoiceCmd::Remove(agent) => ( + "remove", + VoiceRoomCommand::Remove(VoiceAgentRef::try_from(agent)?), + ), + VoiceCmd::Mute(agent) => ( + "set-muted", + VoiceRoomCommand::SetMuted { + agent: VoiceAgentRef::try_from(agent)?, + muted: true, + }, + ), + VoiceCmd::Unmute(agent) => ( + "set-muted", + VoiceRoomCommand::SetMuted { + agent: VoiceAgentRef::try_from(agent)?, + muted: false, + }, + ), + VoiceCmd::SetVoice { agent, voice } => ( + "set-voice", + VoiceRoomCommand::SetVoice { + agent: VoiceAgentRef::try_from(agent)?, + voice: voice.as_str().to_owned(), + }, + ), + VoiceCmd::MuteOutput => ( + "set-output-muted", + VoiceRoomCommand::SetOutputMuted { muted: true }, + ), + VoiceCmd::UnmuteOutput => ( + "set-output-muted", + VoiceRoomCommand::SetOutputMuted { muted: false }, + ), + }; + let owner = require_owner(client, "voice-room commands")?; + let built = build(client.keys(), &owner, command)?; + let response = client.publish_ephemeral_event(built.event).await?; + let relay: serde_json::Value = serde_json::from_str(&response) + .map_err(|error| CliError::Other(format!("invalid relay response: {error}")))?; + println!( + "{}", + json!({ + "accepted": relay["accepted"], + "event_id": relay["event_id"], + "request_id": built.request_id, + "action": action, + "message": "Voice-room command sent to Buzz Desktop.", + }) + ); + Ok(()) +} diff --git a/crates/buzz-cli/src/lib.rs b/crates/buzz-cli/src/lib.rs index f745e7b280..0e81aec7d9 100644 --- a/crates/buzz-cli/src/lib.rs +++ b/crates/buzz-cli/src/lib.rs @@ -3,6 +3,7 @@ mod client; mod commands; mod error; mod validate; +pub mod voice_management; use clap::{Parser, Subcommand}; use client::BuzzClient; @@ -176,6 +177,9 @@ enum Cmd { /// Draft owner-reviewed agent creation and updates #[command(subcommand)] Agents(AgentsCmd), + /// Control agents participating in the owner's active voice room + #[command(subcommand)] + Voice(VoiceCmd), /// Send, read, search, and manage messages #[command(subcommand)] Messages(MessagesCmd), @@ -241,6 +245,71 @@ enum Cmd { Moderation(ModerationCmd), } +#[derive(Subcommand)] +pub enum VoiceCmd { + /// Add an agent to the active voice room + Join(VoiceAgentArgs), + /// Remove an agent from the active voice room + Remove(VoiceAgentArgs), + /// Mute an agent's microphone in the active voice room + Mute(VoiceAgentArgs), + /// Unmute an agent's microphone in the active voice room + Unmute(VoiceAgentArgs), + /// Select an agent's synthesized voice + SetVoice { + #[command(flatten)] + agent: VoiceAgentArgs, + #[arg(long)] + voice: VoiceName, + }, + /// Mute all synthesized voice output + MuteOutput, + /// Unmute all synthesized voice output + UnmuteOutput, +} + +#[derive(Clone, Copy, clap::ValueEnum)] +pub enum VoiceName { + Sol, + Cove, + Ember, + Breeze, + Arbor, + Vale, + Juniper, + Maple, + Spruce, +} + +impl VoiceName { + fn as_str(self) -> &'static str { + match self { + Self::Sol => "sol", + Self::Cove => "cove", + Self::Ember => "ember", + Self::Breeze => "breeze", + Self::Arbor => "arbor", + Self::Vale => "vale", + Self::Juniper => "juniper", + Self::Maple => "maple", + Self::Spruce => "spruce", + } + } +} + +#[derive(clap::Args, Debug, Clone)] +pub struct VoiceAgentArgs { + /// Exact display name of the agent + #[arg(long)] + agent_name: Option, + /// Agent public key (hex) + #[arg(long)] + agent_pubkey: Option, + /// Existing agent session thread UUID + #[arg(long)] + thread_id: Option, +} + #[derive(Clone, Copy, clap::ValueEnum)] pub enum RespondToArg { #[value(name = "owner-only")] @@ -1972,6 +2041,7 @@ async fn run(cli: Cli) -> Result<(), CliError> { match cli.command { Cmd::Agents(sub) => commands::agents::dispatch(sub, &client).await, + Cmd::Voice(sub) => commands::voice::dispatch(sub, &client).await, Cmd::Messages(sub) => commands::messages::dispatch(sub, &client, &cli.format).await, Cmd::Channels(sub) => commands::channels::dispatch(sub, &client, &cli.format).await, Cmd::Canvas(sub) => commands::channels::dispatch_canvas(sub, &client).await, @@ -2100,6 +2170,7 @@ mod tests { "social", "upload", "users", + "voice", "workflows", ]; @@ -2187,6 +2258,18 @@ mod tests { ] ); assert_eq!(names(&cmd, "canvas"), vec!["get", "set"]); + assert_eq!( + names(&cmd, "voice"), + vec![ + "join", + "mute", + "mute-output", + "remove", + "set-voice", + "unmute", + "unmute-output" + ] + ); assert_eq!(names(&cmd, "reactions"), vec!["add", "get", "remove"]); assert_eq!( names(&cmd, "emoji"), diff --git a/crates/buzz-cli/src/voice_management.rs b/crates/buzz-cli/src/voice_management.rs new file mode 100644 index 0000000000..02575eeccf --- /dev/null +++ b/crates/buzz-cli/src/voice_management.rs @@ -0,0 +1,184 @@ +//! Voice-room commands published through owner-encrypted observer frames. + +use buzz_core::observer::{encrypt_observer_payload, OBSERVER_FRAME_TELEMETRY}; +use nostr::{Event, Keys, PublicKey}; +use serde::Serialize; + +use crate::error::CliError; +use crate::VoiceAgentArgs; + +const REQUEST_KIND: &str = "voice_room_command"; +const MAX_NAME_CHARS: usize = 120; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceAgentRef { + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_pubkey: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_id: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "action", rename_all = "kebab-case")] +pub enum VoiceRoomCommand { + Join(VoiceAgentRef), + Remove(VoiceAgentRef), + SetMuted { + #[serde(flatten)] + agent: VoiceAgentRef, + muted: bool, + }, + SetVoice { + #[serde(flatten)] + agent: VoiceAgentRef, + voice: String, + }, + SetOutputMuted { + muted: bool, + }, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceRoomRequest { + #[serde(rename = "type")] + request_type: &'static str, + command: VoiceRoomCommand, + request_id: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ObserverEvent { + seq: u64, + timestamp: String, + kind: &'static str, + agent_index: Option, + channel_id: Option, + session_id: Option, + turn_id: Option, + payload: VoiceRoomRequest, +} + +pub struct BuiltVoiceRequest { + pub event: Event, + pub request_id: String, +} + +impl TryFrom for VoiceAgentRef { + type Error = CliError; + + fn try_from(value: VoiceAgentArgs) -> Result { + fn clean( + value: Option, + label: &str, + max: usize, + ) -> Result, CliError> { + value + .map(|value| { + let value = value.trim(); + if value.is_empty() { + return Err(CliError::Usage(format!("{label} cannot be empty"))); + } + if value.chars().count() > max { + return Err(CliError::Usage(format!( + "{label} is too long (max {max} characters)" + ))); + } + Ok(value.to_owned()) + }) + .transpose() + } + + let reference = Self { + agent_name: clean(value.agent_name, "agent name", MAX_NAME_CHARS)?, + agent_pubkey: clean(value.agent_pubkey, "agent pubkey", 64)?, + thread_id: clean(value.thread_id, "thread id", 128)?, + }; + if reference.agent_name.is_none() + && reference.agent_pubkey.is_none() + && reference.thread_id.is_none() + { + return Err(CliError::Usage( + "provide --agent-name, --agent-pubkey, or --thread-id".into(), + )); + } + Ok(reference) + } +} + +pub fn build( + keys: &Keys, + owner: &PublicKey, + command: VoiceRoomCommand, +) -> Result { + let request_id = uuid::Uuid::new_v4().to_string(); + let payload = ObserverEvent { + seq: 0, + timestamp: chrono::Utc::now().to_rfc3339(), + kind: REQUEST_KIND, + agent_index: None, + channel_id: None, + session_id: None, + turn_id: None, + payload: VoiceRoomRequest { + request_type: REQUEST_KIND, + command, + request_id: request_id.clone(), + }, + }; + let encrypted = encrypt_observer_payload(keys, owner, &payload) + .map_err(|error| CliError::Other(format!("could not encrypt voice command: {error}")))?; + let event = buzz_sdk::build_agent_observer_frame( + &owner.to_hex(), + &keys.public_key().to_hex(), + OBSERVER_FRAME_TELEMETRY, + &encrypted, + ) + .map_err(|error| CliError::Other(format!("could not build voice command: {error}")))? + .sign_with_keys(keys) + .map_err(|error| CliError::Other(format!("could not sign voice command: {error}")))?; + Ok(BuiltVoiceRequest { event, request_id }) +} + +#[cfg(test)] +mod tests { + use super::*; + use buzz_core::observer::decrypt_observer_payload; + + #[test] + fn command_is_owner_encrypted_and_matches_desktop_contract() { + let agent = Keys::generate(); + let owner = Keys::generate(); + let built = build( + &agent, + &owner.public_key(), + VoiceRoomCommand::Join(VoiceAgentRef { + agent_name: Some("Architect".into()), + agent_pubkey: None, + thread_id: None, + }), + ) + .unwrap(); + + let payload: serde_json::Value = decrypt_observer_payload(&owner, &built.event).unwrap(); + assert_eq!(payload["kind"], REQUEST_KIND); + assert_eq!(payload["payload"]["type"], REQUEST_KIND); + assert_eq!(payload["payload"]["command"]["action"], "join"); + assert_eq!(payload["payload"]["command"]["agentName"], "Architect"); + } + + #[test] + fn agent_reference_is_required() { + let error = VoiceAgentRef::try_from(VoiceAgentArgs { + agent_name: None, + agent_pubkey: None, + thread_id: None, + }) + .unwrap_err(); + assert!(error.to_string().contains("provide --agent-name")); + } +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 365c8f712e..6f506d8621 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1078,6 +1078,7 @@ dependencies = [ "notify-rust", "objc2", "objc2-app-kit", + "objc2-av-foundation", "objc2-foundation", "opus", "plist", @@ -6392,6 +6393,18 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.0", + "block2", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-avf-audio" version = "0.3.2" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index bf3a4ffe96..59399a1f1d 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -50,6 +50,7 @@ block2 = { version = "0.6", default-features = false, features = ["std"] } objc2 = { version = "0.6.4", default-features = false } objc2-app-kit = { version = "0.3.2", default-features = false, features = ["NSEvent", "NSHapticFeedback", "NSMenu", "NSMenuItem", "NSStatusItem", "block2"] } objc2-foundation = { version = "0.3.2", default-features = false, features = ["NSProcessInfo", "NSString"] } +objc2-av-foundation = { version = "0.3.2", default-features = false, features = ["AVCaptureDevice", "AVMediaFormat", "block2"] } keyring = { version = "3.6.3", default-features = false, features = ["apple-native", "vendored"], optional = true } security-framework = { version = "3.7.0", features = ["OSX_10_15"] } window-vibrancy = "0.6" diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index fc90e6ab14..9ba042baa4 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -54,6 +54,7 @@ pub struct AppState { pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, pub huddle_audio: crate::huddle::tts_settings::HuddleAudioSettingsState, + pub codex_voice: Mutex, /// Tauri app handle — stored after setup so huddle commands can emit /// `huddle-state-changed` events without needing the handle threaded /// through every call site. @@ -134,7 +135,6 @@ pub struct AppState { /// `is_member=false`. pub pending_owned_channels: Mutex>, } - /// Parse the `BUZZ_PRIVATE_KEY` env var into identity keys. `Some` means the /// env var was present and valid and MUST win over any persisted/keyring key /// (the dev/CI/harness override). `None` means absent or malformed — callers @@ -191,7 +191,6 @@ pub fn build_app_state() -> AppState { } None => (Keys::generate(), IdentityStorage::Ephemeral), }; - AppState { keys: Mutex::new(keys), identity_storage: AtomicU8::new(identity_storage as u8), @@ -218,6 +217,7 @@ pub fn build_app_state() -> AppState { session_config_cache: Mutex::new(HashMap::new()), huddle_state: Mutex::new(HuddleState::default()), huddle_audio: Default::default(), + codex_voice: Mutex::new(crate::codex_voice::CodexVoiceState::default()), app_handle: Mutex::new(None), media_proxy_port: AtomicU16::new(0), prevent_sleep: Arc::new(Mutex::new( diff --git a/desktop/src-tauri/src/codex_voice.rs b/desktop/src-tauri/src/codex_voice.rs new file mode 100644 index 0000000000..73cf8a65f3 --- /dev/null +++ b/desktop/src-tauri/src/codex_voice.rs @@ -0,0 +1,878 @@ +use std::{ + collections::HashMap, + fs, + io::{BufRead, BufReader, Write}, + path::PathBuf, + process::{Child, ChildStdin, Command, Stdio}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc, Arc, Mutex, + }, + time::Duration, +}; + +use serde::Serialize; +use serde_json::{json, Value}; +use tauri::{AppHandle, Emitter, Manager}; + +use crate::managed_agents::{ + known_acp_runtime, load_managed_agents, load_personas, record_agent_command, resolve_command, + KnownAcpRuntime, +}; + +const LINKS_FILE_NAME: &str = "codex-voice-links.json"; +const VOICE_EVENT: &str = "codex-voice-event"; +const REALTIME_MODEL: &str = "gpt-live-1-codex"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const STOP_TIMEOUT: Duration = Duration::from_secs(2); + +#[derive(Default)] +pub struct CodexVoiceState { + sessions: HashMap, +} + +struct CodexVoiceSession { + thread_id: String, + runtime_thread_id: String, + muted: bool, + mode: CodexVoiceMode, + voice: String, + client: CodexAppServerClient, +} + +struct CodexAppServerClient { + child: Child, + stdin: Arc>, + next_id: AtomicU64, + pending: Arc>>>>, + expected_shutdown: Arc, +} + +struct CodexVoiceRuntimeEnv { + private_key_nsec: String, + auth_tag: Option, + relay_url: String, + mode: CodexVoiceMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum CodexVoiceMode { + Native, + Proxy, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexVoiceTargetLink { + channel_id: String, + thread_id: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct CodexVoiceEvent { + method: String, + params: Value, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexVoiceCapability { + supported: bool, + reason: Option, + model: Option, + mode: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexVoiceStartResponse { + muted: bool, + model: String, + mode: CodexVoiceMode, + voice: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexVoiceStatus { + active: bool, + muted: bool, + model: Option, + sessions: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexVoiceSessionStatus { + thread_id: String, + muted: bool, + model: String, + mode: CodexVoiceMode, + voice: String, +} + +fn agent_voice_mode( + app: &AppHandle, + pubkey: &str, + _relay_url: &str, +) -> Result, String> { + let records = load_managed_agents(app)?; + let personas = load_personas(app).unwrap_or_default(); + let Some(record) = records + .iter() + .find(|record| record.pubkey.eq_ignore_ascii_case(pubkey)) + else { + return Ok(None); + }; + let command = record_agent_command(record, &personas); + Ok(Some(resolve_voice_mode( + &record.name, + known_acp_runtime(&command).is_some_and(KnownAcpRuntime::supports_native_voice), + ))) +} + +fn resolve_voice_mode(_agent_name: &str, _supports_native_voice: bool) -> CodexVoiceMode { + CodexVoiceMode::Proxy +} + +fn codex_voice_runtime_env( + app: &AppHandle, + pubkey: &str, + relay_url: &str, +) -> Result, String> { + let records = load_managed_agents(app)?; + let personas = load_personas(app).unwrap_or_default(); + let Some(record) = records + .into_iter() + .find(|record| record.pubkey.eq_ignore_ascii_case(pubkey)) + else { + return Ok(None); + }; + let command = record_agent_command(&record, &personas); + let mode = resolve_voice_mode( + &record.name, + known_acp_runtime(&command).is_some_and(KnownAcpRuntime::supports_native_voice), + ); + if record.private_key_nsec.is_empty() { + return Err("The managed agent identity is unavailable.".to_string()); + } + let workspace_relay = + crate::relay::relay_ws_url_with_override(&app.state::()); + let relay_url = resolve_voice_relay_url(&record.relay_url, relay_url, &workspace_relay); + if relay_url.is_empty() { + return Err("The managed agent relay is unavailable.".to_string()); + } + Ok(Some(CodexVoiceRuntimeEnv { + private_key_nsec: record.private_key_nsec, + auth_tag: record.auth_tag, + relay_url, + mode, + })) +} + +fn resolve_voice_relay_url( + record_relay_url: &str, + requested_relay_url: &str, + workspace_relay_url: &str, +) -> String { + let requested_relay_url = requested_relay_url.trim(); + let candidate = if requested_relay_url.is_empty() { + record_relay_url + } else { + requested_relay_url + }; + crate::relay::effective_agent_relay_url(candidate, workspace_relay_url) +} + +fn configure_voice_runtime_env(command: &mut Command, runtime_env: &CodexVoiceRuntimeEnv) { + command.env("BUZZ_PRIVATE_KEY", &runtime_env.private_key_nsec); + command.env("NOSTR_PRIVATE_KEY", &runtime_env.private_key_nsec); + command.env("BUZZ_RELAY_URL", &runtime_env.relay_url); + if let Some(auth_tag) = &runtime_env.auth_tag { + command.env("BUZZ_AUTH_TAG", auth_tag); + } else { + command.env_remove("BUZZ_AUTH_TAG"); + } +} + +fn realtime_voice(requested: &str) -> &'static str { + match requested { + "arbor" => "arbor", + "breeze" => "breeze", + "cove" => "cove", + "ember" => "ember", + "juniper" => "juniper", + "maple" => "maple", + "spruce" => "spruce", + "vale" => "vale", + _ => "sol", + } +} + +fn voice_links_path(app: &AppHandle) -> Result { + Ok(app + .path() + .app_data_dir() + .map_err(|error| format!("Could not resolve Buzz app data: {error}"))? + .join("agents") + .join(LINKS_FILE_NAME)) +} + +fn voice_link_key(pubkey: &str, channel_id: &str) -> String { + format!("{}:{channel_id}", pubkey.to_ascii_lowercase()) +} + +fn agent_voice_link_key(pubkey: &str) -> String { + voice_link_key(pubkey, "*") +} + +fn agent_voice_target_key(pubkey: &str) -> String { + format!("target:{}", pubkey.to_ascii_lowercase()) +} + +fn load_voice_links(app: &AppHandle) -> Result, String> { + let path = voice_links_path(app)?; + if !path.exists() { + return Ok(HashMap::new()); + } + let bytes = + fs::read(&path).map_err(|error| format!("Could not read Codex Voice links: {error}"))?; + serde_json::from_slice(&bytes) + .map_err(|error| format!("Could not parse Codex Voice links: {error}")) +} + +#[tauri::command] +pub fn get_codex_voice_link( + pubkey: String, + channel_id: String, + app: AppHandle, +) -> Result, String> { + let links = load_voice_links(&app)?; + Ok(links + .get(&voice_link_key(&pubkey, &channel_id)) + .or_else(|| links.get(&agent_voice_link_key(&pubkey))) + .or_else(|| { + let prefix = format!("{}:", pubkey.to_ascii_lowercase()); + links + .iter() + .find_map(|(key, thread_id)| key.starts_with(&prefix).then_some(thread_id)) + }) + .cloned()) +} + +#[tauri::command] +pub fn get_codex_voice_target_link( + pubkey: String, + app: AppHandle, +) -> Result, String> { + let links = load_voice_links(&app)?; + if let Some(encoded) = links.get(&agent_voice_target_key(&pubkey)) { + if let Some((channel_id, thread_id)) = encoded.split_once('\u{001f}') { + return Ok(Some(CodexVoiceTargetLink { + channel_id: channel_id.to_string(), + thread_id: thread_id.to_string(), + })); + } + } + + let prefix = format!("{}:", pubkey.to_ascii_lowercase()); + Ok(links.iter().find_map(|(key, thread_id)| { + let channel_id = key.strip_prefix(&prefix)?; + if channel_id == "*" || channel_id.is_empty() { + return None; + } + Some(CodexVoiceTargetLink { + channel_id: channel_id.to_string(), + thread_id: thread_id.clone(), + }) + })) +} + +#[tauri::command] +pub fn remember_codex_voice_link( + pubkey: String, + channel_id: String, + thread_id: String, + app: AppHandle, +) -> Result<(), String> { + if pubkey.is_empty() || channel_id.is_empty() || thread_id.is_empty() { + return Err("Codex Voice link fields cannot be empty.".to_string()); + } + let path = voice_links_path(&app)?; + let parent = path + .parent() + .ok_or_else(|| "Codex Voice link path has no parent.".to_string())?; + fs::create_dir_all(parent) + .map_err(|error| format!("Could not create Codex Voice link directory: {error}"))?; + let mut links = load_voice_links(&app)?; + links.insert(voice_link_key(&pubkey, &channel_id), thread_id.clone()); + links.insert(agent_voice_link_key(&pubkey), thread_id.clone()); + links.insert( + agent_voice_target_key(&pubkey), + format!("{channel_id}\u{001f}{thread_id}"), + ); + let bytes = serde_json::to_vec_pretty(&links) + .map_err(|error| format!("Could not encode Codex Voice links: {error}"))?; + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes) + .map_err(|error| format!("Could not write Codex Voice links: {error}"))?; + fs::rename(&temporary, &path) + .map_err(|error| format!("Could not save Codex Voice links: {error}")) +} + +#[tauri::command] +pub fn get_codex_voice_capability( + pubkey: String, + relay_url: String, + app: AppHandle, +) -> Result { + let Some(mode) = agent_voice_mode(&app, &pubkey, &relay_url)? else { + return Ok(CodexVoiceCapability { + supported: false, + reason: None, + model: None, + mode: None, + }); + }; + if resolve_command("codex").is_none() { + return Ok(CodexVoiceCapability { + supported: false, + reason: Some("The Codex runtime is not installed.".to_string()), + model: None, + mode: None, + }); + } + Ok(CodexVoiceCapability { + supported: true, + reason: None, + model: Some(REALTIME_MODEL.to_string()), + mode: Some(mode), + }) +} + +#[cfg(target_os = "macos")] +#[tauri::command] +pub fn request_microphone_access() -> Result { + use block2::RcBlock; + use objc2_av_foundation::{AVAuthorizationStatus, AVCaptureDevice, AVMediaTypeAudio}; + + let audio_media_type = unsafe { AVMediaTypeAudio } + .ok_or_else(|| "AVFoundation audio capture is unavailable.".to_string())?; + let status = unsafe { AVCaptureDevice::authorizationStatusForMediaType(audio_media_type) }; + if status == AVAuthorizationStatus::Authorized { + return Ok(true); + } + if status == AVAuthorizationStatus::Denied || status == AVAuthorizationStatus::Restricted { + return Ok(false); + } + + let (sender, receiver) = mpsc::channel(); + let completion = RcBlock::new(move |granted: objc2::runtime::Bool| { + let _ = sender.send(granted.as_bool()); + }); + unsafe { + AVCaptureDevice::requestAccessForMediaType_completionHandler(audio_media_type, &completion); + } + receiver + .recv_timeout(Duration::from_secs(60)) + .map_err(|_| "Microphone permission request timed out.".to_string()) +} + +#[cfg(not(target_os = "macos"))] +#[tauri::command] +pub fn request_microphone_access() -> Result { + Ok(true) +} + +impl CodexAppServerClient { + fn spawn( + app: AppHandle, + thread_id: String, + runtime_env: &CodexVoiceRuntimeEnv, + ) -> Result { + let binary = resolve_command("codex") + .ok_or_else(|| "The Codex runtime could not be found.".to_string())?; + let mut command = Command::new(binary); + configure_voice_runtime_env(&mut command, runtime_env); + let mut child = command + .args(["app-server", "--stdio", "--enable", "realtime_conversation"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| format!("Could not start the Codex voice runtime: {error}"))?; + let stdin = + Arc::new(Mutex::new(child.stdin.take().ok_or_else(|| { + "Codex voice stdin is unavailable.".to_string() + })?)); + let stdout = child + .stdout + .take() + .ok_or_else(|| "Codex voice stdout is unavailable.".to_string())?; + let stderr = child.stderr.take(); + let pending: Arc>>>> = + Arc::new(Mutex::new(HashMap::new())); + let reader_pending = Arc::clone(&pending); + let expected_shutdown = Arc::new(AtomicBool::new(false)); + let reader_expected_shutdown = Arc::clone(&expected_shutdown); + let reader_app = app.clone(); + let reader_thread_id = thread_id; + + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + let Ok(message) = serde_json::from_str::(&line) else { + continue; + }; + if let Some(id) = message.get("id").and_then(Value::as_u64) { + if let Ok(mut waiters) = reader_pending.lock() { + if let Some(sender) = waiters.remove(&id) { + let result = if let Some(error) = message.get("error") { + Err(json_rpc_error(error)) + } else { + Ok(message.get("result").cloned().unwrap_or(Value::Null)) + }; + let _ = sender.send(result); + } + } + continue; + } + let Some(method) = message.get("method").and_then(Value::as_str) else { + continue; + }; + if method.starts_with("thread/realtime/") { + let mut params = message.get("params").cloned().unwrap_or(Value::Null); + if let Some(params) = params.as_object_mut() { + params.insert( + "threadId".to_string(), + Value::String(reader_thread_id.clone()), + ); + } + let _ = reader_app.emit( + VOICE_EVENT, + CodexVoiceEvent { + method: method.to_string(), + params, + }, + ); + } + } + if !reader_expected_shutdown.load(Ordering::Acquire) { + let _ = reader_app.emit( + VOICE_EVENT, + CodexVoiceEvent { + method: "thread/realtime/error".to_string(), + params: json!({ + "threadId": reader_thread_id, + "message": "The Codex voice runtime closed unexpectedly." + }), + }, + ); + } + }); + + if let Some(stderr) = stderr { + std::thread::spawn(move || { + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + eprintln!("buzz-codex-voice: {line}"); + } + }); + } + + Ok(Self { + child, + stdin, + next_id: AtomicU64::new(1), + pending, + expected_shutdown, + }) + } + + fn write(&self, message: &Value) -> Result<(), String> { + let mut stdin = self.stdin.lock().map_err(|error| error.to_string())?; + serde_json::to_writer(&mut *stdin, message) + .map_err(|error| format!("Could not encode a Codex voice request: {error}"))?; + stdin + .write_all(b"\n") + .and_then(|_| stdin.flush()) + .map_err(|error| format!("Could not send a Codex voice request: {error}")) + } + + fn notify(&self, method: &str, params: Value) -> Result<(), String> { + self.write(&json!({ "method": method, "params": params })) + } + + fn request(&self, method: &str, params: Value) -> Result { + self.request_with_timeout(method, params, REQUEST_TIMEOUT) + } + + fn request_with_timeout( + &self, + method: &str, + params: Value, + timeout: Duration, + ) -> Result { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let (sender, receiver) = mpsc::channel(); + self.pending + .lock() + .map_err(|error| error.to_string())? + .insert(id, sender); + if let Err(error) = self.write(&json!({ + "id": id, + "method": method, + "params": params + })) { + if let Ok(mut pending) = self.pending.lock() { + pending.remove(&id); + } + return Err(error); + } + receiver.recv_timeout(timeout).map_err(|_| { + if let Ok(mut pending) = self.pending.lock() { + pending.remove(&id); + } + format!("Codex voice timed out while calling {method}.") + })? + } + + fn stop(mut self, thread_id: &str) { + self.expected_shutdown.store(true, Ordering::Release); + let _ = self.request_with_timeout( + "thread/realtime/stop", + json!({ "threadId": thread_id }), + STOP_TIMEOUT, + ); + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Drop for CodexAppServerClient { + fn drop(&mut self) { + self.expected_shutdown.store(true, Ordering::Release); + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn json_rpc_error(error: &Value) -> String { + error + .get("message") + .and_then(Value::as_str) + .map(ToString::to_string) + .unwrap_or_else(|| error.to_string()) +} + +#[tauri::command] +pub fn get_codex_voice_status(app: AppHandle) -> Result { + let state = app.state::(); + let guard = state + .codex_voice + .lock() + .map_err(|error| error.to_string())?; + let sessions = guard + .sessions + .values() + .map(|session| CodexVoiceSessionStatus { + thread_id: session.thread_id.clone(), + muted: session.muted, + model: REALTIME_MODEL.to_string(), + mode: session.mode, + voice: session.voice.clone(), + }) + .collect::>(); + Ok(CodexVoiceStatus { + active: !sessions.is_empty(), + muted: !sessions.is_empty() && sessions.iter().all(|session| session.muted), + model: (!sessions.is_empty()).then(|| REALTIME_MODEL.to_string()), + sessions, + }) +} + +#[tauri::command] +pub fn start_codex_voice( + thread_id: String, + pubkey: String, + agent_name: String, + relay_url: String, + voice: String, + sdp: String, + app: AppHandle, +) -> Result { + if thread_id.is_empty() || sdp.is_empty() { + return Err("The agent task and WebRTC offer are required.".to_string()); + } + let runtime_env = codex_voice_runtime_env(&app, &pubkey, &relay_url)? + .ok_or_else(|| "This agent is not using the native Codex runtime.".to_string())?; + { + let state = app.state::(); + if state + .codex_voice + .lock() + .map_err(|error| error.to_string())? + .sessions + .contains_key(&thread_id) + { + return Err("Codex Voice is already active for this agent task.".to_string()); + } + } + + let client = CodexAppServerClient::spawn(app.clone(), thread_id.clone(), &runtime_env)?; + client.request( + "initialize", + json!({ + "clientInfo": { + "name": "buzz_desktop", + "title": "Buzz", + "version": env!("CARGO_PKG_VERSION") + }, + "capabilities": { + "experimentalApi": true, + "requestAttestation": false + } + }), + )?; + client.notify("initialized", json!({}))?; + let runtime_thread_id = match runtime_env.mode { + CodexVoiceMode::Native => { + client.request( + "thread/resume", + json!({ + "threadId": thread_id, + "excludeTurns": true, + "approvalPolicy": "never" + }), + )?; + thread_id.clone() + } + CodexVoiceMode::Proxy => client + .request( + "thread/start", + json!({ + "ephemeral": true, + "approvalPolicy": "never", + "developerInstructions": "This ephemeral thread is a speech transport owned by Buzz. Do not run tools or take independent action." + }), + )? + .pointer("/thread/id") + .and_then(Value::as_str) + .map(ToString::to_string) + .ok_or_else(|| "Codex Voice could not create a proxy thread.".to_string())?, + }; + let selected_voice = realtime_voice(&voice).to_string(); + let instructions = match runtime_env.mode { + CodexVoiceMode::Native => format!( + "You are speaking inside Buzz as {agent_name}. This realtime session is attached \ + directly to this managed agent's existing Codex task. Use that task's instructions, \ + workspace, tools, and current context. You are also in a shared voice room: you may \ + hear the user and other agents. Respond only when addressed or when you can add \ + material value; avoid acknowledgements and do not create conversational loops." + ), + CodexVoiceMode::Proxy => format!( + "You are the realtime speech transport for the Buzz agent {agent_name}. Transcribe \ + every incoming user or peer utterance accurately. Never answer, acknowledge, \ + summarize, or speak on your own. The Buzz host supplies that agent's exact replies \ + through speech append operations. Remain silent after every incoming utterance." + ), + }; + client.request( + "thread/realtime/start", + json!({ + "threadId": runtime_thread_id, + "model": REALTIME_MODEL, + "version": "v3", + "voice": selected_voice, + "outputModality": "audio", + "includeStartupContext": runtime_env.mode == CodexVoiceMode::Native, + "flushTranscriptTailOnSessionEnd": true, + "codexResponsesAsItems": runtime_env.mode == CodexVoiceMode::Native, + "codexResponseHandoffMode": "thinking", + "initialItems": [{ + "role": "developer", + "text": instructions + }], + "transport": { + "type": "webrtc", + "sdp": sdp + } + }), + )?; + + app.state::() + .codex_voice + .lock() + .map_err(|error| error.to_string())? + .sessions + .insert( + thread_id.clone(), + CodexVoiceSession { + thread_id, + runtime_thread_id, + muted: false, + mode: runtime_env.mode, + voice: selected_voice.clone(), + client, + }, + ); + + Ok(CodexVoiceStartResponse { + muted: false, + model: REALTIME_MODEL.to_string(), + mode: runtime_env.mode, + voice: selected_voice, + }) +} + +#[tauri::command] +pub fn speak_codex_voice(thread_id: String, text: String, app: AppHandle) -> Result<(), String> { + let text = text.trim(); + if text.is_empty() { + return Ok(()); + } + let state = app.state::(); + let guard = state + .codex_voice + .lock() + .map_err(|error| error.to_string())?; + let session = guard + .sessions + .get(&thread_id) + .ok_or_else(|| "No Codex Voice session is active for this agent task.".to_string())?; + session.client.request( + "thread/realtime/appendSpeech", + json!({ + "threadId": session.runtime_thread_id, + "text": text + }), + )?; + Ok(()) +} + +#[tauri::command] +pub fn set_codex_voice_muted( + thread_id: String, + muted: bool, + app: AppHandle, +) -> Result { + let state = app.state::(); + let mut guard = state + .codex_voice + .lock() + .map_err(|error| error.to_string())?; + let session = guard + .sessions + .get_mut(&thread_id) + .ok_or_else(|| "No Codex Voice session is active for this agent task.".to_string())?; + session.muted = muted; + Ok(muted) +} + +#[tauri::command] +pub fn stop_codex_voice(thread_id: String, app: AppHandle) -> Result<(), String> { + let session = app + .state::() + .codex_voice + .lock() + .map_err(|error| error.to_string())? + .sessions + .remove(&thread_id); + if let Some(session) = session { + session.client.stop(&session.runtime_thread_id); + } + Ok(()) +} + +pub fn shutdown_codex_voice(app: &AppHandle) { + let sessions = app + .state::() + .codex_voice + .lock() + .ok() + .map(|mut state| std::mem::take(&mut state.sessions)) + .unwrap_or_default(); + for session in sessions.into_values() { + session.client.stop(&session.runtime_thread_id); + } +} + +#[cfg(test)] +mod tests { + use super::{ + configure_voice_runtime_env, realtime_voice, resolve_voice_mode, resolve_voice_relay_url, + CodexVoiceMode, CodexVoiceRuntimeEnv, + }; + use std::{collections::HashMap, ffi::OsString, process::Command}; + + fn configured_env(runtime_env: &CodexVoiceRuntimeEnv) -> HashMap> { + let mut command = Command::new("codex"); + configure_voice_runtime_env(&mut command, runtime_env); + command + .get_envs() + .map(|(key, value)| (key.to_os_string(), value.map(OsString::from))) + .collect() + } + + #[test] + fn voice_runtime_receives_managed_agent_buzz_identity() { + let env = configured_env(&CodexVoiceRuntimeEnv { + private_key_nsec: "nsec-test".to_string(), + auth_tag: Some("owner-attestation".to_string()), + relay_url: "wss://relay.example".to_string(), + mode: CodexVoiceMode::Native, + }); + + assert_eq!( + env.get(&OsString::from("BUZZ_PRIVATE_KEY")), + Some(&Some(OsString::from("nsec-test"))) + ); + assert_eq!( + env.get(&OsString::from("NOSTR_PRIVATE_KEY")), + Some(&Some(OsString::from("nsec-test"))) + ); + assert_eq!( + env.get(&OsString::from("BUZZ_RELAY_URL")), + Some(&Some(OsString::from("wss://relay.example"))) + ); + assert_eq!( + env.get(&OsString::from("BUZZ_AUTH_TAG")), + Some(&Some(OsString::from("owner-attestation"))) + ); + } + + #[test] + fn voice_runtime_removes_inherited_auth_tag_for_legacy_agent() { + let env = configured_env(&CodexVoiceRuntimeEnv { + private_key_nsec: "nsec-test".to_string(), + auth_tag: None, + relay_url: "wss://relay.example".to_string(), + mode: CodexVoiceMode::Proxy, + }); + + assert_eq!(env.get(&OsString::from("BUZZ_AUTH_TAG")), Some(&None)); + } + + #[test] + fn voice_runtime_uses_active_workspace_relay_for_unpinned_agent() { + assert_eq!( + resolve_voice_relay_url("", "", "wss://workspace.example"), + "wss://workspace.example" + ); + } + + #[test] + fn unsupported_realtime_voice_falls_back_to_sol() { + assert_eq!(realtime_voice("cove"), "cove"); + assert_eq!(realtime_voice("cedar"), "sol"); + assert_eq!(realtime_voice("not-a-voice"), "sol"); + } + + #[test] + fn voice_uses_proxy_transport_for_every_agent() { + assert_eq!(resolve_voice_mode("Orchestrator", true), CodexVoiceMode::Proxy); + assert_eq!(resolve_voice_mode("Builder", true), CodexVoiceMode::Proxy); + assert_eq!(resolve_voice_mode("Explorer", false), CodexVoiceMode::Proxy); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index c4b733e3e0..a0b75ee01c 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -2,6 +2,7 @@ mod app_state; mod archive; mod builderlab; +mod codex_voice; mod commands; mod deep_link; mod egress_guard; @@ -39,6 +40,7 @@ mod util; pub mod webkit_rendering; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use builderlab::*; +use codex_voice::*; use commands::*; use deep_link::{ acknowledge_pending_community_deep_link, handle_deep_link_url, @@ -108,10 +110,8 @@ async fn clear_initial_window_backing(window: &tauri::Window< async fn wait_for_stable_initial_window_geometry(window: &tauri::Window) { const MAX_POLLS: usize = 120; const REQUIRED_STABLE_POLLS: usize = 4; - let mut previous_bounds = None; let mut stable_polls = 0; - for _ in 0..MAX_POLLS { // Accept whatever geometry the window-state plugin restores — maximized // or a normal saved size. macOS applies the restore asynchronously, so @@ -123,7 +123,6 @@ async fn wait_for_stable_initial_window_geometry(window: &tau (Ok(position), Ok(size)) => Some((position.x, position.y, size.width, size.height)), _ => None, }; - if bounds.is_some() && bounds == previous_bounds { stable_polls += 1; if stable_polls >= REQUIRED_STABLE_POLLS { @@ -133,10 +132,8 @@ async fn wait_for_stable_initial_window_geometry(window: &tau stable_polls = 0; } previous_bounds = bounds; - tokio::time::sleep(std::time::Duration::from_millis(16)).await; } - eprintln!("buzz-desktop: initial window geometry did not settle before reveal timeout"); } @@ -169,7 +166,6 @@ pub fn run() { eprintln!("buzz-mesh: failed to build big-stack tokio runtime, using default: {error}"); } } - let builder = tauri::Builder::default() .plugin(tauri_plugin_single_instance::init(|app, argv, _cwd| { // Focus the existing window when a duplicate instance launches. @@ -199,31 +195,25 @@ pub fn run() { if webview.label() != "main" { return; } - // Linux/WebKitGTK needs media-stream settings and a // permission-request handler for getUserMedia; no-op // on macOS/Windows. linux_media::enable_media_capture(&webview); - // macOS applies the restored geometry asynchronously. Wait // for several identical outer bounds and for React to // commit the startup surface before revealing it. let window = webview.window(); - #[cfg(target_os = "macos")] { set_initial_window_backing(&window); - let (initial_render_tx, initial_render_rx) = tokio::sync::oneshot::channel(); window .app_handle() .once(INITIAL_RENDER_READY_EVENT, move |_| { let _ = initial_render_tx.send(()); }); - tauri::async_runtime::spawn(async move { wait_for_stable_initial_window_geometry(&window).await; - if tokio::time::timeout( std::time::Duration::from_secs(5), initial_render_rx, @@ -909,6 +899,16 @@ pub fn run() { list_audio_output_devices, set_audio_output_device, get_audio_output_device, + get_codex_voice_capability, + get_codex_voice_link, + get_codex_voice_status, + get_codex_voice_target_link, + remember_codex_voice_link, + request_microphone_access, + speak_codex_voice, + start_codex_voice, + stop_codex_voice, + set_codex_voice_muted, start_pairing, confirm_pairing_sas, cancel_pairing, diff --git a/desktop/src-tauri/src/managed_agents/discovery/runtime_metadata.rs b/desktop/src-tauri/src/managed_agents/discovery/runtime_metadata.rs index 34edecdcd9..61a56daa09 100644 --- a/desktop/src-tauri/src/managed_agents/discovery/runtime_metadata.rs +++ b/desktop/src-tauri/src/managed_agents/discovery/runtime_metadata.rs @@ -67,6 +67,10 @@ pub(crate) struct KnownAcpRuntime { } impl KnownAcpRuntime { + pub fn supports_native_voice(&self) -> bool { + self.id == "codex" + } + /// Return the CLI install commands for the current platform. /// /// On Windows, returns `cli_install_commands_windows` when non-empty, diff --git a/desktop/src-tauri/src/managed_agents/nest_skill.md b/desktop/src-tauri/src/managed_agents/nest_skill.md index 79a5ea301d..1a89226a0a 100644 --- a/desktop/src-tauri/src/managed_agents/nest_skill.md +++ b/desktop/src-tauri/src/managed_agents/nest_skill.md @@ -3,7 +3,7 @@ name: buzz-cli description: > Buzz CLI for relay operations: owner-reviewed agent drafts, messaging, channels, DMs, users, workflows, feed, reactions, canvas, social, repos, - uploads, and agent memory. + uploads, voice-room control, and agent memory. version: 1 --- @@ -17,6 +17,8 @@ version: 1 `BUZZ_AUTH_TAG` is required for `buzz agents draft-create` and `buzz agents draft-update` because those commands send owner-reviewed Desktop drafts. If missing, explain that this managed agent cannot open owner-reviewed agent drafts from chat. +`BUZZ_AUTH_TAG` is also required for voice-room commands because they are encrypted to and executed by the owner's Buzz Desktop. + Run the bundled CLI with `--help` and ` --help` to discover all flags, arguments, and usage. This skill documents only what `--help` cannot tell you. ## Conversational Agent Management @@ -41,6 +43,20 @@ buzz agents draft-update --channel --agent-name "Current name" \ Run `buzz agents draft-update --help` for optional runtime, provider, model, rename, and access changes. Prefer these CLI commands over any legacy MCP agent-management tools. +## Voice Room Control + +Control existing personal agents in the owner's currently open Voice room without UI automation: + +```bash +buzz voice join --agent-name "Architect" +buzz voice mute --agent-name "Architect" +buzz voice unmute --agent-name "Architect" +buzz voice set-voice --agent-name "Architect" --voice "verse" +buzz voice remove --agent-name "Architect" +``` + +Use `buzz voice mute-output` and `buzz voice unmute-output` for room-wide synthesized output. A successful CLI response means the relay accepted the encrypted command event; Desktop execution is asynchronous. These commands do not create agents. + ## Git Repositories Buzz hosts real git repos, and **you can own one yourself** — no human key needed. `repos create` signs the announcement with *your* key, so the repo is owned by whoever runs it; the owner segment in the clone URL is your own pubkey (hex, not a username). Git auth is automatic: the harness configures the `git-credential-nostr` helper, so plain `git clone`/`push`/`pull` against `/git//` just work over NIP-98 — never put a private key on a git command line. Announce with `repos create --id --clone /git//`, then `git remote add origin ` and `git push -u origin main` (the relay seeds an empty repo on announce, so it's immediately pushable). Requires git 2.46+ for the credential protocol. diff --git a/desktop/src-tauri/src/shutdown.rs b/desktop/src-tauri/src/shutdown.rs index 95f9efc3c5..58a0fe145d 100644 --- a/desktop/src-tauri/src/shutdown.rs +++ b/desktop/src-tauri/src/shutdown.rs @@ -18,6 +18,7 @@ pub(crate) fn shut_down_app(app: &tauri::AppHandle, shutdown_done: &std::sync::a .shutdown_started .store(true, Ordering::SeqCst); if !shutdown_done.swap(true, Ordering::SeqCst) { + crate::codex_voice::shutdown_codex_voice(app); prevent_sleep::release(&app.state::().prevent_sleep); if let Err(error) = shutdown_managed_agents(app) { eprintln!("buzz-desktop: failed to stop managed agents: {error}"); @@ -40,6 +41,7 @@ pub(crate) fn install_signal_handler( .shutdown_started .store(true, Ordering::SeqCst); if !shutdown_done.swap(true, Ordering::SeqCst) { + crate::codex_voice::shutdown_codex_voice(&app); let _ = shutdown_managed_agents(&app); #[cfg(feature = "mesh-llm")] shutdown_mesh_runtime(&app); diff --git a/desktop/src/app/AppShell.helpers.ts b/desktop/src/app/AppShell.helpers.ts index dd6b9195e8..1c6768a63b 100644 --- a/desktop/src/app/AppShell.helpers.ts +++ b/desktop/src/app/AppShell.helpers.ts @@ -7,6 +7,7 @@ export type AppView = | "channel" | "messages" | "agents" + | "voice" | "workflows" | "pulse" | "projects"; @@ -167,6 +168,13 @@ export function deriveShellRoute(pathname: string): { }; } + if (pathname === "/voice") { + return { + selectedChannelId: null, + selectedView: "voice", + }; + } + if (pathname === "/workflows" || pathname.startsWith("/workflows/")) { return { selectedChannelId: null, diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 46e06b2f9f..02965ac149 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -41,6 +41,7 @@ import { useAutoRestartPolicy } from "@/features/agents/lib/useAutoRestartPolicy import { usePersonaSync } from "@/features/agents/lib/usePersonaSync"; import { useAgentObserverIngestion } from "@/features/agents/useAgentObserverIngestion"; import { AgentManagementDialogs } from "@/features/agents/ui/AgentManagementDialogs"; +import { CodexVoiceController } from "@/features/agents/ui/CodexVoiceController"; import { RequestedAgentCreateDialogs } from "@/features/agents/ui/RequestedAgentCreateDialogs"; import { usePresenceSession, @@ -130,6 +131,7 @@ export function AppShell() { goNewMessage, goProjects, goPulse, + goVoice, goSettings, goWorkflows, closeSettings, @@ -253,13 +255,11 @@ export function AppShell() { return; } hasRestoredCommunityDestinationRef.current = true; - // Restoration belongs to an explicit community transition. Cold boot and // reconnect remounts must preserve the route the user explicitly opened. if (!consumePendingCommunityRestore(activeCommunityId)) { return; } - const destination = loadCommunityDestination(activeCommunityId); if (!destination || destination.kind === "home") { return; @@ -273,7 +273,6 @@ export function AppShell() { void goHome({ replace: true }); return; } - // The normal switch path writes the remembered channel into the hash before // the target community mounts, so no intermediate Inbox frame is painted. // Older transition callers may still arrive at neutral Home; repair those. @@ -314,7 +313,6 @@ export function AppShell() { openSearchHit, pubkey: identityQuery.data?.pubkey, }); - const { followedRootIds, isFollowing: isFollowingThread, @@ -884,6 +882,7 @@ export function AppShell() { onSelectHome={() => void goHome()} onSelectProjects={() => void goProjects()} onSelectPulse={() => void goPulse()} + onSelectVoice={() => void goVoice()} onSelectSettings={handleOpenSettings} onSelectWorkflows={() => void goWorkflows()} onSetPresenceStatus={(status) => @@ -940,6 +939,7 @@ export function AppShell() { /> )} + + commitNavigation( + { + to: "/voice", + }, + behavior, + ), + [commitNavigation], + ); + const goPulse = React.useCallback( (behavior?: NavigationBehavior) => commitNavigation( @@ -308,6 +319,7 @@ export function useAppNavigation() { closeSettings, closeWorkflowDetail, goAgents, + goVoice, goChannel, goForumPost, goHome, diff --git a/desktop/src/app/routeTree.gen.ts b/desktop/src/app/routeTree.gen.ts index 2bc2c8ddb6..a254ece610 100644 --- a/desktop/src/app/routeTree.gen.ts +++ b/desktop/src/app/routeTree.gen.ts @@ -6,6 +6,7 @@ import { Route as rootRouteImport } from "./routes/root"; import { Route as workflowsRouteImport } from "./routes/workflows"; +import { Route as voiceRouteImport } from "./routes/voice"; import { Route as settingsRouteImport } from "./routes/settings"; import { Route as remindersRouteImport } from "./routes/reminders"; import { Route as pulseRouteImport } from "./routes/pulse"; @@ -23,6 +24,11 @@ const workflowsRoute = workflowsRouteImport.update({ path: "/workflows", getParentRoute: () => rootRouteImport, } as any); +const voiceRoute = voiceRouteImport.update({ + id: "/voice", + path: "/voice", + getParentRoute: () => rootRouteImport, +} as any); const settingsRoute = settingsRouteImport.update({ id: "/settings", path: "/settings", @@ -87,6 +93,7 @@ export interface FileRoutesByFullPath { "/pulse": typeof pulseRoute; "/reminders": typeof remindersRoute; "/settings": typeof settingsRoute; + "/voice": typeof voiceRoute; "/workflows": typeof workflowsRoute; "/channels/$channelId": typeof channelsDotchannelIdRoute; "/messages/new": typeof messagesDotnewRoute; @@ -101,6 +108,7 @@ export interface FileRoutesByTo { "/pulse": typeof pulseRoute; "/reminders": typeof remindersRoute; "/settings": typeof settingsRoute; + "/voice": typeof voiceRoute; "/workflows": typeof workflowsRoute; "/channels/$channelId": typeof channelsDotchannelIdRoute; "/messages/new": typeof messagesDotnewRoute; @@ -116,6 +124,7 @@ export interface FileRoutesById { "/pulse": typeof pulseRoute; "/reminders": typeof remindersRoute; "/settings": typeof settingsRoute; + "/voice": typeof voiceRoute; "/workflows": typeof workflowsRoute; "/channels/$channelId": typeof channelsDotchannelIdRoute; "/messages/new": typeof messagesDotnewRoute; @@ -132,6 +141,7 @@ export interface FileRouteTypes { | "/pulse" | "/reminders" | "/settings" + | "/voice" | "/workflows" | "/channels/$channelId" | "/messages/new" @@ -146,6 +156,7 @@ export interface FileRouteTypes { | "/pulse" | "/reminders" | "/settings" + | "/voice" | "/workflows" | "/channels/$channelId" | "/messages/new" @@ -160,6 +171,7 @@ export interface FileRouteTypes { | "/pulse" | "/reminders" | "/settings" + | "/voice" | "/workflows" | "/channels/$channelId" | "/messages/new" @@ -175,6 +187,7 @@ export interface RootRouteChildren { pulseRoute: typeof pulseRoute; remindersRoute: typeof remindersRoute; settingsRoute: typeof settingsRoute; + voiceRoute: typeof voiceRoute; workflowsRoute: typeof workflowsRoute; channelsDotchannelIdRoute: typeof channelsDotchannelIdRoute; messagesDotnewRoute: typeof messagesDotnewRoute; @@ -192,6 +205,13 @@ declare module "@tanstack/react-router" { preLoaderRoute: typeof workflowsRouteImport; parentRoute: typeof rootRouteImport; }; + "/voice": { + id: "/voice"; + path: "/voice"; + fullPath: "/voice"; + preLoaderRoute: typeof voiceRouteImport; + parentRoute: typeof rootRouteImport; + }; "/settings": { id: "/settings"; path: "/settings"; @@ -279,6 +299,7 @@ const rootRouteChildren: RootRouteChildren = { pulseRoute: pulseRoute, remindersRoute: remindersRoute, settingsRoute: settingsRoute, + voiceRoute: voiceRoute, workflowsRoute: workflowsRoute, channelsDotchannelIdRoute: channelsDotchannelIdRoute, messagesDotnewRoute: messagesDotnewRoute, diff --git a/desktop/src/app/routes.ts b/desktop/src/app/routes.ts index f5c6938e11..6fd9565a77 100644 --- a/desktop/src/app/routes.ts +++ b/desktop/src/app/routes.ts @@ -3,6 +3,7 @@ import { index, rootRoute, route } from "@tanstack/virtual-file-routes"; export const routes = rootRoute("root.tsx", [ index("index.tsx"), route("/agents", "agents.tsx"), + route("/voice", "voice.tsx"), route("/pulse", "pulse.tsx"), route("/reminders", "reminders.tsx"), route("/settings", "settings.tsx"), diff --git a/desktop/src/app/routes/voice.tsx b/desktop/src/app/routes/voice.tsx new file mode 100644 index 0000000000..21de5d6292 --- /dev/null +++ b/desktop/src/app/routes/voice.tsx @@ -0,0 +1,21 @@ +import * as React from "react"; +import { createFileRoute } from "@tanstack/react-router"; + +import { ViewLoadingFallback } from "@/shared/ui/ViewLoadingFallback"; + +const VoiceModeScreen = React.lazy(async () => { + const module = await import("@/features/agents/ui/VoiceModeScreen"); + return { default: module.VoiceModeScreen }; +}); + +export const Route = createFileRoute("/voice")({ + component: VoiceRouteComponent, +}); + +function VoiceRouteComponent() { + return ( + }> + + + ); +} diff --git a/desktop/src/features/agents/observerRelayStore.ts b/desktop/src/features/agents/observerRelayStore.ts index 56c69f915a..37535eb21e 100644 --- a/desktop/src/features/agents/observerRelayStore.ts +++ b/desktop/src/features/agents/observerRelayStore.ts @@ -11,6 +11,10 @@ import { parseAgentManagementRequest, type AgentManagementRequest, } from "./agentManagement"; +import { + parseVoiceRoomCommandRequest, + type VoiceRoomCommandRequest, +} from "./voiceRoomService"; import { normalizePubkey } from "@/shared/lib/pubkey"; import { useQueryClient } from "@tanstack/react-query"; import { agentConfigSurfaceQueryKey } from "@/features/agents/hooks"; @@ -104,6 +108,9 @@ const controlResultListeners = new Map< const agentManagementListeners = new Set< (agentPubkey: string, request: AgentManagementRequest) => void >(); +const voiceRoomCommandListeners = new Set< + (agentPubkey: string, request: VoiceRoomCommandRequest) => void +>(); // Normalized pubkeys of agents we are actively managing. Only events whose // "agent" tag matches an entry here will be decrypted (defense-in-depth). @@ -398,6 +405,12 @@ async function handleRelayObserverEvent( listener(agentPubkey, managementRequest); } } + const voiceRoomRequest = parseVoiceRoomCommandRequest(parsed.payload); + if (voiceRoomRequest) { + for (const listener of voiceRoomCommandListeners) { + listener(agentPubkey, voiceRoomRequest); + } + } if (parsed.kind === "session_config_captured") { void putAgentSessionConfig(agentPubkey, parsed.payload); onSessionConfigCaptured?.(agentPubkey); @@ -522,6 +535,15 @@ export function subscribeAgentManagementRequests( }; } +export function subscribeVoiceRoomCommandRequests( + listener: (agentPubkey: string, request: VoiceRoomCommandRequest) => void, +) { + voiceRoomCommandListeners.add(listener); + return () => { + voiceRoomCommandListeners.delete(listener); + }; +} + export function subscribeControlResults( agentPubkey: string, listener: (frame: ControlResultFrame) => void, @@ -750,6 +772,7 @@ export function resetAgentObserverStore() { pendingUnknownAgentFrames.length = 0; latestLiveSessionByAgentChannel.clear(); agentManagementListeners.clear(); + voiceRoomCommandListeners.clear(); onSessionConfigCaptured = null; connectionState = "idle"; errorMessage = null; diff --git a/desktop/src/features/agents/ui/CodexVoiceController.tsx b/desktop/src/features/agents/ui/CodexVoiceController.tsx new file mode 100644 index 0000000000..b2bc6825b5 --- /dev/null +++ b/desktop/src/features/agents/ui/CodexVoiceController.tsx @@ -0,0 +1,213 @@ +import { Mic } from "lucide-react"; +import * as React from "react"; +import { useLocation } from "@tanstack/react-router"; + +import { CodexVoiceDock } from "@/features/agents/ui/CodexVoiceDock"; +import { useManagedAgentsQuery } from "@/features/agents/hooks"; +import { subscribeVoiceRoomCommandRequests } from "@/features/agents/observerRelayStore"; +import { + endVoiceTarget, + hasVoiceTarget, + startVoiceTarget, + useCodexVoiceTargets, + useSavedCodexVoiceTargets, + type CodexVoiceTargetInput, +} from "@/features/agents/voiceSessionRegistry"; +import { + type CodexVoiceMode, + getCodexVoiceLinkRevision, + getCodexVoiceCapability, + getCodexVoiceTargetLink, + subscribeCodexVoiceLinkChanges, +} from "@/shared/api/codexVoice"; +import { Button } from "@/shared/ui/button"; +import { + executeVoiceRoomCommand, + installVoiceRoomCommandBridge, + updateVoiceRoomCommandContext, +} from "@/features/agents/voiceRoomService"; + +export function CodexVoiceController() { + const activeTargets = useCodexVoiceTargets(); + const savedTargets = useSavedCodexVoiceTargets(); + const agentsQuery = useManagedAgentsQuery(); + const voiceLinkRevision = React.useSyncExternalStore( + subscribeCodexVoiceLinkChanges, + getCodexVoiceLinkRevision, + getCodexVoiceLinkRevision, + ); + const [availableTargets, setAvailableTargets] = React.useState< + CodexVoiceTargetInput[] + >([]); + const location = useLocation(); + const isVoiceMode = location.pathname === "/voice"; + React.useEffect(() => installVoiceRoomCommandBridge(), []); + React.useEffect( + () => + subscribeVoiceRoomCommandRequests((_agentPubkey, request) => { + executeVoiceRoomCommand(request.command); + }), + [], + ); + React.useEffect(() => { + void voiceLinkRevision; + let cancelled = false; + const agents = agentsQuery.data ?? []; + void Promise.all( + agents + .filter( + (agent) => agent.name.trim().toLowerCase() !== "github buzz sync", + ) + .map(async (agent): Promise => { + const saved = savedTargets.find( + (target) => + target.agentPubkey.toLowerCase() === agent.pubkey.toLowerCase(), + ); + const [capability, link] = await Promise.all([ + getCodexVoiceCapability(agent.pubkey, agent.relayUrl), + getCodexVoiceTargetLink(agent.pubkey), + ]); + const resolvedLink = + link ?? + (saved + ? { channelId: saved.channelId, threadId: saved.threadId } + : null); + const mode = capability.mode ?? saved?.mode ?? null; + if (!capability.supported || !mode || !resolvedLink) return null; + return { + agentName: agent.name, + agentPubkey: agent.pubkey, + channelId: resolvedLink.channelId, + mode, + relayUrl: agent.relayUrl, + threadId: resolvedLink.threadId, + voice: saved?.voice, + } satisfies CodexVoiceTargetInput; + }), + ).then((targets) => { + if (!cancelled) { + setAvailableTargets( + targets.filter( + (target): target is CodexVoiceTargetInput => target !== null, + ), + ); + } + }); + return () => { + cancelled = true; + }; + }, [agentsQuery.data, savedTargets, voiceLinkRevision]); + React.useEffect(() => { + updateVoiceRoomCommandContext({ activeTargets, availableTargets }); + }, [activeTargets, availableTargets]); + return ( +
+ {activeTargets.map((target) => ( + + ))} +
+ ); +} + +type CodexVoiceLauncherProps = { + agentName: string; + agentPubkey: string; + channelId: string | null; + isWorking: boolean; + relayUrl: string; + threadId: string | null; +}; + +export function CodexVoiceLauncher({ + agentName, + agentPubkey, + channelId, + isWorking, + relayUrl, + threadId, +}: CodexVoiceLauncherProps) { + const activeTargets = useCodexVoiceTargets(); + const [supported, setSupported] = React.useState(false); + const [model, setModel] = React.useState("gpt-live-1-codex"); + const [mode, setMode] = React.useState(null); + + React.useEffect(() => { + if (!threadId || !relayUrl) { + setSupported(false); + setMode(null); + return; + } + let cancelled = false; + void getCodexVoiceCapability(agentPubkey, relayUrl) + .then((capability) => { + if (cancelled) return; + setSupported(capability.supported); + setMode(capability.mode); + if (capability.model) setModel(capability.model); + }) + .catch(() => { + if (!cancelled) { + setSupported(false); + setMode(null); + } + }); + return () => { + cancelled = true; + }; + }, [agentPubkey, relayUrl, threadId]); + + if (!supported || !threadId || !channelId || !mode) return null; + + const active = hasVoiceTarget(activeTargets, threadId); + const disabled = isWorking || active; + + return ( +
+ +
+ ); +} diff --git a/desktop/src/features/agents/ui/CodexVoiceDock.tsx b/desktop/src/features/agents/ui/CodexVoiceDock.tsx new file mode 100644 index 0000000000..4526aae733 --- /dev/null +++ b/desktop/src/features/agents/ui/CodexVoiceDock.tsx @@ -0,0 +1,667 @@ +import { listen } from "@tauri-apps/api/event"; +import { + Captions, + LoaderCircle, + Mic, + MicOff, + PhoneOff, + RotateCcw, + X, +} from "lucide-react"; +import * as React from "react"; + +import { voiceRoomAudio } from "@/features/agents/voiceRoomAudio"; +import { + appendVoiceRoomTranscript, + releaseVoiceRoomSpeaker, + routeVoiceRoomTurn, + setVoiceTargetMuted, + type CodexVoiceTarget, + updateVoiceSessionState, + useCodexVoiceTargets, + useVoiceRoomOutputMuted, + useVoiceRoomDirectedTurns, + useVoiceRoomSpeakerLease, +} from "@/features/agents/voiceSessionRegistry"; +import { getThreadReference } from "@/features/messages/lib/threading"; +import { + type CodexVoiceEvent, + requestMicrophoneAccess, + setCodexVoiceMuted, + speakCodexVoice, + startCodexVoice, + stopCodexVoice, +} from "@/shared/api/codexVoice"; +import { relayClient } from "@/shared/api/relayClient"; +import { sendChannelMessage } from "@/shared/api/tauri"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; + +type VoicePhase = "starting" | "listening" | "ending" | "error"; + +type MeterBar = { + id: string; + value: number; +}; + +type VoiceTranscriptEntry = { + id: number; + role: "assistant" | "user"; + text: string; +}; + +type CodexVoiceDockProps = { + onEnded: (threadId: string) => void; + target: CodexVoiceTarget; +}; + +const DEFAULT_LEVELS: MeterBar[] = [ + { id: "outer-left", value: 0.18 }, + { id: "inner-left", value: 0.28 }, + { id: "center", value: 0.2 }, + { id: "inner-right", value: 0.34 }, + { id: "outer-right", value: 0.16 }, +]; + +const NATIVE_RESPONSE_TIMEOUT_MS = 45_000; + +export function CodexVoiceDock({ onEnded, target }: CodexVoiceDockProps) { + const { agentName, agentPubkey, channelId, mode, relayUrl, threadId, voice } = + target; + const [model, setModel] = React.useState("gpt-live-1-codex"); + const roomOutputMuted = useVoiceRoomOutputMuted(); + const activeTargets = useCodexVoiceTargets(); + const directedTurns = useVoiceRoomDirectedTurns(); + const speakerLease = useVoiceRoomSpeakerLease(); + const isOrchestrator = agentName.trim().toLowerCase() === "orchestrator"; + const capturesRoomInput = + isOrchestrator || + (activeTargets.length === 1 && activeTargets[0]?.threadId === threadId); + const [phase, setPhase] = React.useState("starting"); + const [error, setError] = React.useState(null); + const [muted, setMuted] = React.useState(false); + const [transcript, setTranscript] = React.useState(null); + const [transcriptExpanded, setTranscriptExpanded] = React.useState(false); + const [transcriptHistory, setTranscriptHistory] = React.useState< + VoiceTranscriptEntry[] + >([]); + const [levels, setLevels] = React.useState(DEFAULT_LEVELS); + const streamRef = React.useRef(null); + const peerRef = React.useRef(null); + const remoteAudioRef = React.useRef(null); + const analyserContextRef = React.useRef(null); + const analyserFrameRef = React.useRef(null); + const phaseRef = React.useRef(phase); + const startedRef = React.useRef(false); + const proxyRootIdsRef = React.useRef(new Set()); + const spokenProxyMessagesRef = React.useRef(new Set()); + const proxySpeechQueueRef = React.useRef(Promise.resolve()); + const pendingDirectedReplyRef = React.useRef(false); + const recoveringRef = React.useRef(false); + const responseWatchdogRef = React.useRef(null); + const restartVoiceRef = React.useRef<() => void>(() => undefined); + const transcriptSequenceRef = React.useRef(0); + const handledDirectedTurnRef = React.useRef(0); + const speakerLeaseRef = React.useRef(speakerLease); + const transportMutedRef = React.useRef(null); + phaseRef.current = phase; + speakerLeaseRef.current = speakerLease; + + React.useEffect(() => { + updateVoiceSessionState(threadId, { + error, + muted, + phase, + transcript, + }); + }, [error, muted, phase, threadId, transcript]); + + const clearResponseWatchdog = React.useCallback(() => { + if (responseWatchdogRef.current === null) return; + window.clearTimeout(responseWatchdogRef.current); + responseWatchdogRef.current = null; + }, []); + + const releaseMedia = React.useCallback(() => { + clearResponseWatchdog(); + if (analyserFrameRef.current !== null) { + cancelAnimationFrame(analyserFrameRef.current); + analyserFrameRef.current = null; + } + if (analyserContextRef.current) { + void analyserContextRef.current.close(); + analyserContextRef.current = null; + } + peerRef.current?.close(); + peerRef.current = null; + voiceRoomAudio.leave(threadId); + releaseVoiceRoomSpeaker(threadId); + streamRef.current = null; + if (remoteAudioRef.current) remoteAudioRef.current.srcObject = null; + setMuted(false); + setTranscript(null); + setTranscriptExpanded(false); + setTranscriptHistory([]); + setLevels(DEFAULT_LEVELS); + }, [clearResponseWatchdog, threadId]); + + React.useEffect(() => { + if (mode !== "proxy") return; + let disposed = false; + let unsubscribe: (() => Promise) | undefined; + void relayClient + .subscribeToChannelLive(channelId, (event) => { + if ( + disposed || + event.pubkey.toLowerCase() !== agentPubkey.toLowerCase() || + spokenProxyMessagesRef.current.has(event.id) + ) { + return; + } + const reference = getThreadReference(event.tags); + const belongsToVoiceTurn = + (reference.rootId && proxyRootIdsRef.current.has(reference.rootId)) || + (reference.parentId && + proxyRootIdsRef.current.has(reference.parentId)); + if ( + (!belongsToVoiceTurn && !pendingDirectedReplyRef.current) || + !event.content.trim() + ) { + return; + } + + pendingDirectedReplyRef.current = false; + spokenProxyMessagesRef.current.add(event.id); + proxySpeechQueueRef.current = proxySpeechQueueRef.current + .catch(() => undefined) + .then(() => speakCodexVoice(threadId, event.content)) + .catch((speechError) => { + if (!disposed) { + releaseVoiceRoomSpeaker(threadId); + setError( + formatError( + speechError, + `${agentName}'s voice reply could not play.`, + ), + ); + } + }); + }) + .then((dispose) => { + if (disposed) { + void dispose(); + } else { + unsubscribe = dispose; + } + }) + .catch((subscriptionError) => { + if (!disposed) { + setError( + formatError( + subscriptionError, + `Buzz could not listen for ${agentName}'s replies.`, + ), + ); + } + }); + return () => { + disposed = true; + if (unsubscribe) void unsubscribe(); + }; + }, [agentName, agentPubkey, channelId, mode, threadId]); + + const sendProxyTurn = React.useCallback( + (text: string) => { + pendingDirectedReplyRef.current = true; + return sendChannelMessage(channelId, text, null, undefined, [agentPubkey]) + .then((result) => { + proxyRootIdsRef.current.add(result.rootEventId ?? result.eventId); + }) + .catch((sendError) => { + pendingDirectedReplyRef.current = false; + releaseVoiceRoomSpeaker(threadId); + setError( + formatError( + sendError, + `${agentName} could not receive the voice turn.`, + ), + ); + }); + }, + [agentName, agentPubkey, channelId, threadId], + ); + + React.useEffect(() => { + if (mode !== "proxy") return; + const turn = directedTurns.at(-1); + if ( + !turn || + turn.id <= handledDirectedTurnRef.current || + turn.recipientThreadId !== threadId + ) { + return; + } + handledDirectedTurnRef.current = turn.id; + void sendProxyTurn(turn.text); + }, [directedTurns, mode, sendProxyTurn, threadId]); + + const removeDock = React.useCallback(() => { + phaseRef.current = "ending"; + releaseMedia(); + onEnded(threadId); + }, [onEnded, releaseMedia, threadId]); + + const finishVoice = React.useCallback(async () => { + if (phaseRef.current === "ending") return; + phaseRef.current = "ending"; + setPhase("ending"); + try { + await stopCodexVoice(threadId); + } finally { + releaseMedia(); + onEnded(threadId); + } + }, [onEnded, releaseMedia, threadId]); + + React.useEffect(() => { + let cancelled = false; + const unlisten = listen("codex-voice-event", (event) => { + if ( + cancelled || + (event.payload.params.threadId && + event.payload.params.threadId !== threadId) + ) { + return; + } + const { method, params } = event.payload; + if (method === "thread/realtime/sdp" && params.sdp) { + void peerRef.current + ?.setRemoteDescription({ type: "answer", sdp: params.sdp }) + .catch((rtcError) => { + setPhase("error"); + setError(formatError(rtcError, "Voice audio could not connect.")); + }); + } else if (method === "thread/realtime/started") { + recoveringRef.current = false; + setPhase("listening"); + } else if (method === "thread/realtime/transcript/delta") { + setTranscript((current) => `${current ?? ""}${params.delta ?? ""}`); + } else if (method === "thread/realtime/transcript/done") { + const completedTranscript = params.text?.trim() || null; + setTranscript(completedTranscript); + if (completedTranscript) { + const role = params.role === "assistant" ? "assistant" : "user"; + const ownsFloor = + role !== "assistant" || + !speakerLeaseRef.current || + speakerLeaseRef.current.threadId === threadId; + if ( + (role === "user" && isOrchestrator) || + (role === "assistant" && ownsFloor) + ) { + appendVoiceRoomTranscript({ + speakerName: role === "assistant" ? agentName : "You", + speakerType: role === "assistant" ? "agent" : "human", + text: completedTranscript, + }); + } + transcriptSequenceRef.current += 1; + setTranscriptHistory((current) => [ + ...current.slice(-49), + { + id: transcriptSequenceRef.current, + role, + text: completedTranscript, + }, + ]); + } + if (params.role === "assistant") { + clearResponseWatchdog(); + if (speakerLeaseRef.current?.threadId === threadId) { + window.setTimeout(() => releaseVoiceRoomSpeaker(threadId), 250); + } + } else if (params.text?.trim()) { + if (isOrchestrator) { + routeVoiceRoomTurn(params.text.trim()); + } else if (capturesRoomInput) { + void sendProxyTurn(params.text.trim()); + } + if (mode === "native" && isOrchestrator) { + clearResponseWatchdog(); + responseWatchdogRef.current = window.setTimeout(() => { + responseWatchdogRef.current = null; + if (phaseRef.current === "ending") return; + recoveringRef.current = true; + setPhase("starting"); + setError("Orchestrator took too long. Reconnecting voice…"); + releaseVoiceRoomSpeaker(threadId); + void stopCodexVoice(threadId).finally(() => { + if (phaseRef.current !== "ending") restartVoiceRef.current(); + }); + }, NATIVE_RESPONSE_TIMEOUT_MS); + } + } + } else if (method === "thread/realtime/error") { + recoveringRef.current = false; + setPhase("error"); + setError(params.message || "Codex Voice encountered an error."); + releaseMedia(); + void stopCodexVoice(threadId); + } else if (method === "thread/realtime/closed") { + if (recoveringRef.current) return; + void stopCodexVoice(threadId).finally(removeDock); + } + }); + return () => { + cancelled = true; + void unlisten.then((dispose) => dispose()); + }; + }, [ + agentName, + capturesRoomInput, + clearResponseWatchdog, + isOrchestrator, + mode, + releaseMedia, + removeDock, + sendProxyTurn, + threadId, + ]); + + React.useEffect( + () => () => { + if (phaseRef.current !== "ending") void stopCodexVoice(threadId); + releaseMedia(); + }, + [releaseMedia, threadId], + ); + + const startLevelMeter = React.useCallback((stream: MediaStream) => { + const context = new AudioContext(); + const source = context.createMediaStreamSource(stream); + const analyser = context.createAnalyser(); + analyser.fftSize = 64; + source.connect(analyser); + const samples = new Uint8Array(analyser.frequencyBinCount); + analyserContextRef.current = context; + const draw = () => { + analyser.getByteFrequencyData(samples); + const average = + samples.reduce((total, sample) => total + sample, 0) / + Math.max(1, samples.length) / + 255; + const weights = [0.45, 0.72, 1, 0.66, 0.4]; + setLevels((current) => + current.map((bar, index) => ({ + ...bar, + value: 0.14 + average * (weights[index] ?? 0.4), + })), + ); + analyserFrameRef.current = requestAnimationFrame(draw); + }; + draw(); + }, []); + + const beginVoice = React.useCallback(async () => { + setError(null); + setTranscript(null); + setPhase("starting"); + phaseRef.current = "starting"; + try { + const microphoneAllowed = await requestMicrophoneAccess(); + if (!microphoneAllowed) { + throw new Error( + "Enable Buzz in System Settings → Privacy & Security → Microphone.", + ); + } + const stream = await voiceRoomAudio.join(threadId); + streamRef.current = stream; + voiceRoomAudio.setMuted(threadId, !capturesRoomInput); + startLevelMeter(stream); + + const peer = new RTCPeerConnection(); + peerRef.current = peer; + for (const track of stream.getAudioTracks()) peer.addTrack(track, stream); + peer.createDataChannel("oai-events"); + peer.ontrack = ({ streams }) => { + const [remoteStream] = streams; + if (!remoteStream || !remoteAudioRef.current) return; + voiceRoomAudio.setRemoteStream(threadId, remoteStream); + remoteAudioRef.current.srcObject = remoteStream; + void remoteAudioRef.current.play(); + }; + peer.onconnectionstatechange = () => { + if (peer.connectionState === "failed") { + setPhase("error"); + setError("The live voice connection failed."); + releaseMedia(); + void stopCodexVoice(threadId); + } + }; + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer); + await waitForIceGathering(peer); + const sdp = peer.localDescription?.sdp; + if (!sdp) throw new Error("Buzz could not create the voice connection."); + + const response = await startCodexVoice({ + threadId, + pubkey: agentPubkey, + agentName, + relayUrl, + voice, + sdp, + }); + setMuted(response.muted); + setModel(response.model); + } catch (startError) { + recoveringRef.current = false; + releaseMedia(); + void stopCodexVoice(threadId); + setPhase("error"); + setError(formatError(startError, "Codex Voice could not start.")); + } + }, [ + agentName, + agentPubkey, + capturesRoomInput, + relayUrl, + releaseMedia, + startLevelMeter, + threadId, + voice, + ]); + restartVoiceRef.current = () => { + releaseMedia(); + void beginVoice(); + }; + + React.useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + void beginVoice(); + }, [beginVoice]); + + React.useEffect(() => { + if (phase !== "listening") return; + const userMuted = target.muted ?? false; + const transportMuted = + userMuted || + !capturesRoomInput || + Boolean(speakerLease && speakerLease.threadId !== threadId); + if (transportMutedRef.current === transportMuted) return; + transportMutedRef.current = transportMuted; + voiceRoomAudio.setMuted(threadId, transportMuted); + setMuted(userMuted); + void setCodexVoiceMuted(threadId, transportMuted).catch((muteError) => { + transportMutedRef.current = null; + voiceRoomAudio.setMuted(threadId, false); + setMuted(false); + setVoiceTargetMuted(threadId, false); + setError(formatError(muteError, "Could not control the microphone.")); + }); + }, [capturesRoomInput, phase, speakerLease, target.muted, threadId]); + + function toggleMute() { + setVoiceTargetMuted(threadId, !muted); + } + + async function retryVoice() { + await stopCodexVoice(threadId); + releaseMedia(); + await beginVoice(); + } + + return ( +
+
+ ); +} + +function VoiceOrb({ levels, muted }: { levels: MeterBar[]; muted: boolean }) { + return ( + + ); +} + +function waitForIceGathering(peer: RTCPeerConnection): Promise { + if (peer.iceGatheringState === "complete") return Promise.resolve(); + return new Promise((resolve) => { + const onStateChange = () => { + if (peer.iceGatheringState !== "complete") return; + peer.removeEventListener("icegatheringstatechange", onStateChange); + resolve(); + }; + peer.addEventListener("icegatheringstatechange", onStateChange); + }); +} + +function formatError(error: unknown, fallback: string): string { + if (typeof error === "string" && error.trim()) return error; + if (error instanceof Error && error.message.trim()) return error.message; + return fallback; +} diff --git a/desktop/src/features/agents/ui/VoiceModeScreen.tsx b/desktop/src/features/agents/ui/VoiceModeScreen.tsx new file mode 100644 index 0000000000..8e3782b2a5 --- /dev/null +++ b/desktop/src/features/agents/ui/VoiceModeScreen.tsx @@ -0,0 +1,795 @@ +import { + Activity, + Bot, + Captions, + Headphones, + LoaderCircle, + Mic, + MicOff, + PhoneOff, + Play, + Radio, + Settings2, + UsersRound, + Volume2, + VolumeX, +} from "lucide-react"; +import * as React from "react"; + +import { useManagedAgentsQuery } from "@/features/agents/hooks"; +import { + saveVoiceTargetPreference, + useCodexVoiceSessionStates, + useCodexVoiceTargets, + useSavedCodexVoiceTargets, + useVoiceRoomOutputMuted, + useVoiceRoomTranscript, + VOICE_ROOM_PALETTE, + type CodexVoiceTarget, + type CodexVoiceTargetInput, + type VoiceRoomTranscriptEntry, +} from "@/features/agents/voiceSessionRegistry"; +import { ManagedAgentSessionPanel } from "@/features/agents/ui/ManagedAgentSessionPanel"; +import { + executeVoiceRoomCommand, + updateVoiceRoomCommandContext, +} from "@/features/agents/voiceRoomService"; +import { + getCodexVoiceLinkRevision, + getCodexVoiceCapability, + getCodexVoiceTargetLink, + subscribeCodexVoiceLinkChanges, +} from "@/shared/api/codexVoice"; +import type { ManagedAgent } from "@/shared/api/types"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { PageHeader, SubsectionLabel } from "@/shared/ui/PageHeader"; + +type VoiceAvailability = + | { kind: "loading" } + | { kind: "ready"; target: CodexVoiceTargetInput } + | { kind: "unavailable"; reason: string }; + +export function VoiceModeScreen() { + const agentsQuery = useManagedAgentsQuery(); + const activeTargets = useCodexVoiceTargets(); + const savedTargets = useSavedCodexVoiceTargets(); + const sessionStates = useCodexVoiceSessionStates(); + const roomTranscript = useVoiceRoomTranscript(); + const roomOutputMuted = useVoiceRoomOutputMuted(); + const voiceLinkRevision = React.useSyncExternalStore( + subscribeCodexVoiceLinkChanges, + getCodexVoiceLinkRevision, + getCodexVoiceLinkRevision, + ); + const [availability, setAvailability] = React.useState< + Record + >({}); + const [selectedPubkey, setSelectedPubkey] = React.useState( + null, + ); + const agents = React.useMemo( + () => agentsQuery.data ?? [], + [agentsQuery.data], + ); + const rosterAgents = React.useMemo( + () => buildVoiceRoster(agents, activeTargets, savedTargets, availability), + [activeTargets, agents, availability, savedTargets], + ); + + React.useEffect(() => { + void voiceLinkRevision; + let cancelled = false; + setAvailability( + Object.fromEntries( + agents.map((agent) => [ + agent.pubkey.toLowerCase(), + { kind: "loading" } satisfies VoiceAvailability, + ]), + ), + ); + for (const agent of agents) { + const key = agent.pubkey.toLowerCase(); + const saved = savedTargets.find( + (target) => target.agentPubkey.toLowerCase() === key, + ); + void Promise.all([ + getCodexVoiceCapability(agent.pubkey, agent.relayUrl), + getCodexVoiceTargetLink(agent.pubkey), + ]) + .then(([capability, link]) => { + if (cancelled) return; + const resolvedLink = + link ?? + (saved + ? { channelId: saved.channelId, threadId: saved.threadId } + : null); + const mode = capability.mode ?? saved?.mode ?? null; + const result: VoiceAvailability = + capability.supported && mode && resolvedLink + ? { + kind: "ready", + target: { + agentName: agent.name, + agentPubkey: agent.pubkey, + channelId: resolvedLink.channelId, + mode, + relayUrl: agent.relayUrl, + threadId: resolvedLink.threadId, + voice: saved?.voice, + }, + } + : { + kind: "unavailable", + reason: + capability.reason ?? + (capability.supported + ? "Start one task with this agent to establish voice context." + : "Voice is unavailable for this agent."), + }; + setAvailability((current) => ({ ...current, [key]: result })); + }) + .catch((error) => { + if (cancelled) return; + setAvailability((current) => ({ + ...current, + [key]: { + kind: "unavailable", + reason: + error instanceof Error + ? error.message + : "Voice capability could not be checked.", + }, + })); + }); + } + return () => { + cancelled = true; + }; + }, [agents, savedTargets, voiceLinkRevision]); + + React.useEffect(() => { + if ( + selectedPubkey && + rosterAgents.some( + (agent) => agent.pubkey.toLowerCase() === selectedPubkey.toLowerCase(), + ) + ) { + return; + } + setSelectedPubkey( + activeTargets[0]?.agentPubkey ?? rosterAgents[0]?.pubkey ?? null, + ); + }, [activeTargets, rosterAgents, selectedPubkey]); + + const selectedAgent = rosterAgents.find( + (agent) => agent.pubkey.toLowerCase() === selectedPubkey?.toLowerCase(), + ); + const selectedTarget = activeTargets.find( + (target) => + target.agentPubkey.toLowerCase() === selectedPubkey?.toLowerCase(), + ); + const selectedState = selectedTarget + ? sessionStates[selectedTarget.threadId] + : null; + const selectedCapability = selectedAgent + ? availability[selectedAgent.pubkey.toLowerCase()] + : null; + const selectedSaved = selectedAgent + ? savedTargets.find( + (target) => + target.agentPubkey.toLowerCase() === + selectedAgent.pubkey.toLowerCase(), + ) + : null; + const selectedVoice = + selectedTarget?.voice ?? + (selectedCapability?.kind === "ready" + ? selectedCapability.target.voice + : null) ?? + selectedSaved?.voice ?? + VOICE_ROOM_PALETTE[0]; + const roomMuted = + activeTargets.length > 0 && activeTargets.every((target) => target.muted); + const activeSpeaker = + activeTargets.find( + (target) => sessionStates[target.threadId]?.phase === "listening", + ) ?? activeTargets[0]; + const activeSpeakerState = activeSpeaker + ? sessionStates[activeSpeaker.threadId] + : null; + + React.useEffect(() => { + updateVoiceRoomCommandContext({ + activeTargets, + availableTargets: Object.values(availability).flatMap((entry) => + entry.kind === "ready" ? [entry.target] : [], + ), + }); + }, [activeTargets, availability]); + + return ( +
+
+
+ + +
+ +
+ + +
+ + +
+ + +
+
+
+ ); +} + +function ParticipantRail({ + activeTargets, + agents, + availability, + onSelect, + selectedPubkey, + sessionStates, +}: { + activeTargets: readonly CodexVoiceTarget[]; + agents: ManagedAgent[]; + availability: Record; + onSelect: (pubkey: string) => void; + selectedPubkey: string | null; + sessionStates: ReturnType; +}) { + return ( + + ); +} + +function VoiceStage({ + activeSpeaker, + activeSpeakerState, + agentCount, +}: { + activeSpeaker: CodexVoiceTarget | undefined; + activeSpeakerState: + | ReturnType[string] + | null; + agentCount: number; +}) { + const listening = activeSpeakerState?.phase === "listening"; + return ( +
+
+
+ + + + + +
+
+ + {agentCount} {agentCount === 1 ? "agent" : "agents"} in the room +
+

+ {activeSpeaker ? activeSpeaker.agentName : "Room ready"} +

+

+ {activeSpeakerState?.error ?? + (activeSpeaker + ? `${activeSpeakerState?.muted ? "Muted" : "Listening"} · ${activeSpeaker.voice} voice` + : "Invite an agent from the team rail to begin. The room follows you throughout Buzz.")} +

+
+ ); +} + +function VoiceConversation({ + entries, +}: { + entries: readonly VoiceRoomTranscriptEntry[]; +}) { + const scrollRef = React.useRef(null); + const [followingLatest, setFollowingLatest] = React.useState(true); + const entryCount = entries.length; + const latestEntryId = entries.at(-1)?.id ?? null; + React.useEffect(() => { + if (latestEntryId === null) return; + const scroller = scrollRef.current; + if (followingLatest && scroller) scroller.scrollTop = scroller.scrollHeight; + }, [followingLatest, latestEntryId]); + const jumpToLatest = React.useCallback(() => { + const scroller = scrollRef.current; + if (!scroller) return; + scroller.scrollTop = scroller.scrollHeight; + setFollowingLatest(true); + }, []); + return ( +
+
+ Conversation +
+ {!followingLatest ? ( + + ) : null} + + + {entryCount} {entryCount === 1 ? "turn" : "turns"} + +
+
+
{ + const scroller = event.currentTarget; + const distanceFromBottom = + scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight; + setFollowingLatest(distanceFromBottom < 48); + }} + ref={scrollRef} + role="log" + > + {entries.length ? ( +
+ {entries.map((entry) => ( +
+
+
+ + {entry.speakerName} + + +
+

+ {entry.text} +

+
+
+ ))} +
+ ) : ( +
+
+ + Voice turns from you and every agent will appear here. +
+
+ )} +
+
+ ); +} + +function AgentInspector({ + agent, + availability, + selectedVoice, + sessionState, + target, +}: { + agent: ManagedAgent | undefined; + availability: VoiceAvailability | null | undefined; + selectedVoice: string; + sessionState: ReturnType[string] | null; + target: CodexVoiceTarget | undefined; +}) { + if (!agent) { + return ( + + ); + } + const readyTarget = + availability?.kind === "ready" ? availability.target : null; + return ( + + ); +} + +function RoomControls({ + activeTargets, + outputMuted, + roomMuted, +}: { + activeTargets: readonly CodexVoiceTarget[]; + outputMuted: boolean; + roomMuted: boolean; +}) { + if (!activeTargets.length) return null; + return ( +
+ + + +
+ ); +} + +function Metric({ label, value }: { label: string; value: string }) { + return ( +
+ {label} + + {value} + +
+ ); +} + +function voiceStatus( + active: CodexVoiceTarget | undefined, + state: ReturnType[string] | null, + availability: VoiceAvailability | undefined, +) { + if (active) { + if (state?.error) return "Connection issue"; + if (active.muted || state?.muted) return "Muted"; + return state?.phase === "listening" ? "Listening" : "Connecting"; + } + if (availability?.kind === "loading") return "Checking voice"; + if (availability?.kind === "unavailable") return availability.reason; + return "Ready to join"; +} + +function buildVoiceRoster( + agents: ManagedAgent[], + activeTargets: readonly CodexVoiceTarget[], + savedTargets: readonly CodexVoiceTarget[], + availability: Readonly>, +) { + const activePubkeys = new Set( + activeTargets.map((target) => target.agentPubkey.toLowerCase()), + ); + const savedPubkeys = new Set( + savedTargets.map((target) => target.agentPubkey.toLowerCase()), + ); + const grouped = new Map(); + for (const agent of agents) { + if (agent.name.trim().toLowerCase() === "github buzz sync") continue; + const groupKey = agent.personaId + ? `persona:${agent.personaId}` + : `name:${agent.name.trim().toLowerCase()}`; + const current = grouped.get(groupKey); + if (!current || agentPriority(agent) > agentPriority(current)) { + grouped.set(groupKey, agent); + } + } + return [...grouped.values()].sort((left, right) => { + const leftActive = activePubkeys.has(left.pubkey.toLowerCase()) ? 1 : 0; + const rightActive = activePubkeys.has(right.pubkey.toLowerCase()) ? 1 : 0; + if (leftActive !== rightActive) return rightActive - leftActive; + const leftSaved = savedPubkeys.has(left.pubkey.toLowerCase()) ? 1 : 0; + const rightSaved = savedPubkeys.has(right.pubkey.toLowerCase()) ? 1 : 0; + if (leftSaved !== rightSaved) return rightSaved - leftSaved; + return left.name.localeCompare(right.name); + }); + + function agentPriority(agent: ManagedAgent) { + const key = agent.pubkey.toLowerCase(); + if (activePubkeys.has(key)) return 5; + if (availability[key]?.kind === "ready") return 4; + if (savedPubkeys.has(key)) return 3; + if (agent.status === "running") return 2; + return agent.status === "deployed" ? 1 : 0; + } +} diff --git a/desktop/src/features/agents/useOpenAgentActivity.ts b/desktop/src/features/agents/useOpenAgentActivity.ts index e8cfc0e8ff..afed357ecc 100644 --- a/desktop/src/features/agents/useOpenAgentActivity.ts +++ b/desktop/src/features/agents/useOpenAgentActivity.ts @@ -7,10 +7,12 @@ import { useAgentSession } from "@/shared/context/AgentSessionContext"; import type { Channel } from "@/shared/api/types"; import { normalizePubkey } from "@/shared/lib/pubkey"; import { getAgentWorkingState } from "./agentWorkingSignal"; -import { useRelayAgentsQuery } from "./hooks"; +import { useManagedAgentsQuery, useRelayAgentsQuery } from "./hooks"; const INACCESSIBLE_ACTIVITY_MESSAGE = "This agent is active in a channel you haven't joined, so its activity can't be opened from here."; +const NO_ACTIVITY_DESTINATION_MESSAGE = + "Open a shared channel or DM with this agent to view its activity."; /** * Can the viewer actually open this channel? Joined channels always; @@ -77,6 +79,7 @@ export function useOpenAgentActivity() { const { goChannel } = useAppNavigation(); const relayAgentsQuery = useRelayAgentsQuery(); const relayAgents = relayAgentsQuery.data; + const managedAgents = useManagedAgentsQuery().data; const channelsQuery = useChannelsQuery(); const channels = channelsQuery.data; @@ -92,13 +95,37 @@ export function useOpenAgentActivity() { const relayAgent = relayAgents?.find( (agent) => normalizePubkey(agent.pubkey) === key, ); + const isLocalAgent = managedAgents?.some( + (agent) => normalizePubkey(agent.pubkey) === key, + ); + const localChannelIds = isLocalAgent + ? (channels ?? []) + .filter((channel) => + channel.memberPubkeys.some( + (memberPubkey) => normalizePubkey(memberPubkey) === key, + ), + ) + .sort((left, right) => { + const leftIsDirectDm = + left.channelType === "dm" && + left.participantPubkeys.length <= 2; + const rightIsDirectDm = + right.channelType === "dm" && + right.participantPubkeys.length <= 2; + return Number(rightIsDirectDm) - Number(leftIsDirectDm); + }) + .map((channel) => channel.id) + : []; const openableChannelIds = new Set( (channels ?? []) .filter((channel) => isChannelOpenable(channel)) .map((channel) => channel.id), ); return resolveOpenableActivityChannelId({ - agentChannelIds: relayAgent?.channelIds ?? [], + agentChannelIds: [ + ...localChannelIds, + ...(relayAgent?.channelIds ?? []), + ], openableChannelIds, // Deliberately an unsubscribed snapshot: this callback runs on click // (and in canOpenAgentActivity), not in render, so we don't need to @@ -111,7 +138,7 @@ export function useOpenAgentActivity() { ), }); }, - [channels, relayAgents], + [channels, managedAgents, relayAgents], ); const canOpenAgentActivity = React.useCallback( @@ -168,6 +195,8 @@ export function useOpenAgentActivity() { // room, or navigating into it. if (getAgentWorkingState(pubkey).channels.length > 0) { toast.warning(INACCESSIBLE_ACTIVITY_MESSAGE); + } else { + toast.info(NO_ACTIVITY_DESTINATION_MESSAGE); } return false; }, diff --git a/desktop/src/features/agents/voiceRoomAudio.test.mjs b/desktop/src/features/agents/voiceRoomAudio.test.mjs new file mode 100644 index 0000000000..8438088ba2 --- /dev/null +++ b/desktop/src/features/agents/voiceRoomAudio.test.mjs @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { mixMinusRecipients } from "./voiceRoomAudio.ts"; + +test("mix-minus routes a speaker to every other room participant", () => { + assert.deepEqual(mixMinusRecipients(["medium", "high", "composer"], "high"), [ + "medium", + "composer", + ]); +}); + +test("mix-minus never feeds a participant its own voice", () => { + assert.deepEqual(mixMinusRecipients(["medium"], "medium"), []); +}); diff --git a/desktop/src/features/agents/voiceRoomAudio.ts b/desktop/src/features/agents/voiceRoomAudio.ts new file mode 100644 index 0000000000..d2aa43d52a --- /dev/null +++ b/desktop/src/features/agents/voiceRoomAudio.ts @@ -0,0 +1,131 @@ +type VoiceRoomParticipant = { + destination: MediaStreamAudioDestinationNode; + remoteSource: MediaStreamAudioSourceNode | null; +}; + +export function mixMinusRecipients( + participantIds: readonly string[], + speakerId: string, +): string[] { + return participantIds.filter((participantId) => participantId !== speakerId); +} + +class VoiceRoomAudioRouter { + private context: AudioContext | null = null; + private microphoneStream: MediaStream | null = null; + private microphoneSource: MediaStreamAudioSourceNode | null = null; + private initialization: Promise | null = null; + private readonly participants = new Map(); + + async join(participantId: string): Promise { + await this.ensureAudioGraph(); + const existing = this.participants.get(participantId); + if (existing) return existing.destination.stream; + + const context = this.context; + const microphoneSource = this.microphoneSource; + if (!context || !microphoneSource) { + throw new Error("Buzz could not initialize the shared voice room."); + } + if (context.state === "suspended") await context.resume(); + + const destination = context.createMediaStreamDestination(); + microphoneSource.connect(destination); + for (const [speakerId, participant] of this.participants) { + if (speakerId !== participantId) { + participant.remoteSource?.connect(destination); + } + } + this.participants.set(participantId, { + destination, + remoteSource: null, + }); + return destination.stream; + } + + setRemoteStream(participantId: string, stream: MediaStream) { + const participant = this.participants.get(participantId); + const context = this.context; + if (!participant || !context || stream.getAudioTracks().length === 0) { + return; + } + + participant.remoteSource?.disconnect(); + const source = context.createMediaStreamSource(stream); + participant.remoteSource = source; + for (const recipientId of mixMinusRecipients( + [...this.participants.keys()], + participantId, + )) { + const recipient = this.participants.get(recipientId); + if (recipient) source.connect(recipient.destination); + } + } + + setMuted(participantId: string, muted: boolean) { + const participant = this.participants.get(participantId); + for (const track of participant?.destination.stream.getAudioTracks() ?? + []) { + track.enabled = !muted; + } + } + + leave(participantId: string) { + const participant = this.participants.get(participantId); + if (!participant) return; + + participant.remoteSource?.disconnect(); + this.microphoneSource?.disconnect(participant.destination); + for (const other of this.participants.values()) { + if (other === participant) continue; + try { + other.remoteSource?.disconnect(participant.destination); + } catch { + // The source may not have been connected to a participant that joined later. + } + } + for (const track of participant.destination.stream.getTracks()) + track.stop(); + this.participants.delete(participantId); + + if (this.participants.size === 0) { + this.microphoneSource?.disconnect(); + for (const track of this.microphoneStream?.getTracks() ?? []) + track.stop(); + this.microphoneSource = null; + this.microphoneStream = null; + const context = this.context; + this.context = null; + if (context) void context.close(); + } + } + + private async ensureAudioGraph() { + if (this.context && this.microphoneStream && this.microphoneSource) return; + if (this.initialization) { + await this.initialization; + return; + } + + this.initialization = (async () => { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + const context = new AudioContext(); + this.microphoneStream = stream; + this.context = context; + this.microphoneSource = context.createMediaStreamSource(stream); + })(); + try { + await this.initialization; + } finally { + this.initialization = null; + } + } +} + +export const voiceRoomAudio = new VoiceRoomAudioRouter(); diff --git a/desktop/src/features/agents/voiceRoomService.test.mjs b/desktop/src/features/agents/voiceRoomService.test.mjs new file mode 100644 index 0000000000..280b35a550 --- /dev/null +++ b/desktop/src/features/agents/voiceRoomService.test.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { getVoiceRoomSnapshot } from "./voiceSessionRegistry.ts"; +import { + executeVoiceRoomCommand, + parseVoiceRoomCommandRequest, + updateVoiceRoomCommandContext, +} from "./voiceRoomService.ts"; + +const architect = { + agentName: "Architect", + agentPubkey: "architect-pubkey", + channelId: "architect-dm", + mode: "proxy", + relayUrl: "wss://relay.example", + threadId: "architect-thread", + voice: "cove", +}; + +test("controls an agent through the application room service", () => { + updateVoiceRoomCommandContext({ + activeTargets: [], + availableTargets: [architect], + }); + + assert.deepEqual( + executeVoiceRoomCommand({ action: "join", agentName: "architect" }), + { + ok: true, + action: "join", + threadId: architect.threadId, + }, + ); + assert.equal(getVoiceRoomSnapshot().activeTargets.length, 1); + + updateVoiceRoomCommandContext({ + activeTargets: getVoiceRoomSnapshot().activeTargets, + availableTargets: [architect], + }); + assert.equal( + executeVoiceRoomCommand({ + action: "set-muted", + agentPubkey: architect.agentPubkey, + muted: true, + }).ok, + true, + ); + assert.equal(getVoiceRoomSnapshot().activeTargets[0]?.muted, true); + + updateVoiceRoomCommandContext({ + activeTargets: getVoiceRoomSnapshot().activeTargets, + availableTargets: [architect], + }); + assert.equal( + executeVoiceRoomCommand({ + action: "set-voice", + threadId: architect.threadId, + voice: "ember", + }).ok, + true, + ); + assert.equal(getVoiceRoomSnapshot().activeTargets[0]?.voice, "ember"); + + executeVoiceRoomCommand({ action: "set-output-muted", muted: true }); + assert.equal(getVoiceRoomSnapshot().outputMuted, true); + + updateVoiceRoomCommandContext({ + activeTargets: getVoiceRoomSnapshot().activeTargets, + availableTargets: [architect], + }); + assert.equal( + executeVoiceRoomCommand({ action: "remove", agentName: "Architect" }).ok, + true, + ); + assert.equal(getVoiceRoomSnapshot().activeTargets.length, 0); +}); + +test("accepts only the narrow agent voice-room command envelope", () => { + assert.deepEqual( + parseVoiceRoomCommandRequest({ + type: "voice_room_command", + requestId: "request-1", + command: { action: "join", agentName: "Architect" }, + }), + { + type: "voice_room_command", + requestId: "request-1", + command: { action: "join", agentName: "Architect" }, + }, + ); + assert.equal( + parseVoiceRoomCommandRequest({ + type: "voice_room_command", + requestId: "request-2", + command: { action: "remove", agentName: "Architect", shell: "rm" }, + }), + null, + ); +}); + +test("rejects an unsupported voice instead of reporting a silent success", () => { + updateVoiceRoomCommandContext({ + activeTargets: [architect], + availableTargets: [architect], + }); + assert.deepEqual( + executeVoiceRoomCommand({ + action: "set-voice", + agentName: "Architect", + voice: "unknown", + }), + { + ok: false, + action: "set-voice", + error: "Voice is not supported.", + }, + ); +}); diff --git a/desktop/src/features/agents/voiceRoomService.ts b/desktop/src/features/agents/voiceRoomService.ts new file mode 100644 index 0000000000..2dba428125 --- /dev/null +++ b/desktop/src/features/agents/voiceRoomService.ts @@ -0,0 +1,184 @@ +import { + endVoiceTarget, + saveVoiceTargetPreference, + setVoiceRoomOutputMuted, + setVoiceTargetMuted, + setVoiceTargetVoice, + startVoiceTarget, + type CodexVoiceTarget, + type CodexVoiceTargetInput, + VOICE_ROOM_PALETTE, +} from "@/features/agents/voiceSessionRegistry"; + +export type VoiceRoomAgentRef = { + agentName?: string; + agentPubkey?: string; + threadId?: string; +}; + +export type VoiceRoomCommand = + | ({ action: "join" } & VoiceRoomAgentRef) + | ({ action: "remove" } & VoiceRoomAgentRef) + | ({ action: "set-muted"; muted: boolean } & VoiceRoomAgentRef) + | ({ action: "set-voice"; voice: string } & VoiceRoomAgentRef) + | { action: "set-output-muted"; muted: boolean }; + +export type VoiceRoomCommandResult = + | { ok: true; action: VoiceRoomCommand["action"]; threadId?: string } + | { ok: false; action: VoiceRoomCommand["action"]; error: string }; + +export const VOICE_ROOM_COMMAND_EVENT = "buzz:voice-room-command"; +export const VOICE_ROOM_COMMAND_RESULT_EVENT = "buzz:voice-room-command-result"; +export const VOICE_ROOM_COMMAND_REQUEST = "voice_room_command"; + +export type VoiceRoomCommandRequest = { + type: typeof VOICE_ROOM_COMMAND_REQUEST; + command: VoiceRoomCommand; + requestId: string; +}; + +export function parseVoiceRoomCommandRequest( + value: unknown, +): VoiceRoomCommandRequest | null { + if (!value || typeof value !== "object") return null; + const request = value as Record; + if ( + request.type !== VOICE_ROOM_COMMAND_REQUEST || + typeof request.requestId !== "string" || + !request.requestId.trim() || + !request.command || + typeof request.command !== "object" + ) { + return null; + } + const command = request.command as Record; + const action = command.action; + if ( + action !== "join" && + action !== "remove" && + action !== "set-muted" && + action !== "set-voice" && + action !== "set-output-muted" + ) { + return null; + } + const allowed = + action === "set-output-muted" + ? ["action", "muted"] + : action === "set-muted" + ? ["action", "agentName", "agentPubkey", "threadId", "muted"] + : action === "set-voice" + ? ["action", "agentName", "agentPubkey", "threadId", "voice"] + : ["action", "agentName", "agentPubkey", "threadId"]; + if (Object.keys(command).some((key) => !allowed.includes(key))) return null; + const hasAgentRef = [ + command.agentName, + command.agentPubkey, + command.threadId, + ].some((candidate) => typeof candidate === "string" && candidate.trim()); + if (action !== "set-output-muted" && !hasAgentRef) return null; + if ( + (action === "set-muted" || action === "set-output-muted") && + typeof command.muted !== "boolean" + ) { + return null; + } + if ( + action === "set-voice" && + (typeof command.voice !== "string" || !command.voice.trim()) + ) { + return null; + } + return request as unknown as VoiceRoomCommandRequest; +} + +let availableTargets: CodexVoiceTargetInput[] = []; +let activeTargets: readonly CodexVoiceTarget[] = []; + +export function updateVoiceRoomCommandContext(input: { + activeTargets: readonly CodexVoiceTarget[]; + availableTargets: readonly CodexVoiceTargetInput[]; +}) { + activeTargets = input.activeTargets; + availableTargets = [...input.availableTargets]; +} + +export function executeVoiceRoomCommand( + command: VoiceRoomCommand, +): VoiceRoomCommandResult { + if (command.action === "set-output-muted") { + setVoiceRoomOutputMuted(command.muted); + return { ok: true, action: command.action }; + } + + const active = findAgent(activeTargets, command); + const available = findAgent(availableTargets, command); + + if (command.action === "join") { + if (active) + return { ok: true, action: command.action, threadId: active.threadId }; + if (!available) + return failure(command, "Agent is not available for voice."); + startVoiceTarget(available); + return { ok: true, action: command.action, threadId: available.threadId }; + } + + if (!active) + return failure(command, "Agent is not active in the voice room."); + + if (command.action === "remove") { + endVoiceTarget(active.threadId); + } else if (command.action === "set-muted") { + setVoiceTargetMuted(active.threadId, command.muted); + } else if (command.action === "set-voice") { + if ( + !VOICE_ROOM_PALETTE.includes( + command.voice as (typeof VOICE_ROOM_PALETTE)[number], + ) + ) { + return failure(command, "Voice is not supported."); + } + setVoiceTargetVoice(active.threadId, command.voice); + saveVoiceTargetPreference({ ...active, voice: command.voice }); + } + return { ok: true, action: command.action, threadId: active.threadId }; +} + +export function installVoiceRoomCommandBridge() { + if (typeof window === "undefined") return () => undefined; + const handleCommand = (event: Event) => { + const request = (event as CustomEvent).detail; + const parsed = parseVoiceRoomCommandRequest(request); + if (!parsed) return; + const result = executeVoiceRoomCommand(parsed.command); + window.dispatchEvent( + new CustomEvent(VOICE_ROOM_COMMAND_RESULT_EVENT, { + detail: { requestId: parsed.requestId, result }, + }), + ); + }; + window.addEventListener(VOICE_ROOM_COMMAND_EVENT, handleCommand); + return () => + window.removeEventListener(VOICE_ROOM_COMMAND_EVENT, handleCommand); +} + +function findAgent( + targets: readonly T[], + reference: VoiceRoomAgentRef, +): T | undefined { + const name = reference.agentName?.trim().toLowerCase(); + const pubkey = reference.agentPubkey?.trim().toLowerCase(); + return targets.find( + (target) => + (reference.threadId && target.threadId === reference.threadId) || + (pubkey && target.agentPubkey.toLowerCase() === pubkey) || + (name && target.agentName.trim().toLowerCase() === name), + ); +} + +function failure( + command: VoiceRoomCommand, + error: string, +): VoiceRoomCommandResult { + return { ok: false, action: command.action, error }; +} diff --git a/desktop/src/features/agents/voiceSessionRegistry.test.mjs b/desktop/src/features/agents/voiceSessionRegistry.test.mjs new file mode 100644 index 0000000000..37cdcfb260 --- /dev/null +++ b/desktop/src/features/agents/voiceSessionRegistry.test.mjs @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + addVoiceTarget, + chooseAvailableVoice, + endVoiceTarget, + getVoiceRoomSnapshot, + hasVoiceTarget, + releaseVoiceRoomSpeaker, + removeVoiceTarget, + routeVoiceRoomTurn, + startVoiceTarget, +} from "./voiceSessionRegistry.ts"; + +const solMedium = { + agentName: "Sol [Medium]", + agentPubkey: "medium", + channelId: "dm-medium", + mode: "native", + relayUrl: "wss://relay.example", + threadId: "thread-medium", + voice: "sol", +}; + +const solHigh = { + agentName: "Sol [High]", + agentPubkey: "high", + channelId: "dm-high", + mode: "native", + relayUrl: "wss://relay.example", + threadId: "thread-high", + voice: "cove", +}; + +test("keeps an active voice target independent of route ownership", () => { + const active = addVoiceTarget([], solMedium); + + assert.equal(hasVoiceTarget(active, solMedium.threadId), true); + assert.deepEqual(active, [solMedium]); +}); + +test("tracks concurrent voice targets for different agent tasks", () => { + const active = addVoiceTarget(addVoiceTarget([], solMedium), solHigh); + + assert.deepEqual(active, [solMedium, solHigh]); +}); + +test("ending one voice target leaves the other session active", () => { + const active = addVoiceTarget(addVoiceTarget([], solMedium), solHigh); + + assert.deepEqual(removeVoiceTarget(active, solMedium.threadId), [solHigh]); +}); + +test("assigns a distinct room voice before reusing the palette", () => { + assert.equal(chooseAvailableVoice([]), "sol"); + assert.equal(chooseAvailableVoice([solMedium]), "cove"); + assert.equal(chooseAvailableVoice([solMedium, solHigh]), "ember"); +}); + +test("assigns one recipient and one speaker lease per room turn", () => { + startVoiceTarget(solMedium); + startVoiceTarget(solHigh); + const turn = routeVoiceRoomTurn("Sol High, please review this"); + + assert.equal(turn?.recipientThreadId, solHigh.threadId); + assert.deepEqual(getVoiceRoomSnapshot().speakerLease, { + threadId: solHigh.threadId, + turnId: turn?.id, + }); + + releaseVoiceRoomSpeaker(solHigh.threadId); + assert.equal(getVoiceRoomSnapshot().speakerLease, null); + endVoiceTarget(solMedium.threadId); + endVoiceTarget(solHigh.threadId); +}); diff --git a/desktop/src/features/agents/voiceSessionRegistry.ts b/desktop/src/features/agents/voiceSessionRegistry.ts new file mode 100644 index 0000000000..0212ce9de6 --- /dev/null +++ b/desktop/src/features/agents/voiceSessionRegistry.ts @@ -0,0 +1,380 @@ +import * as React from "react"; + +import type { CodexVoiceMode } from "@/shared/api/codexVoice"; +import { resolveVoiceTurnRecipient } from "@/features/agents/voiceTurnRouting"; + +export type CodexVoiceTarget = { + agentName: string; + agentPubkey: string; + channelId: string; + mode: CodexVoiceMode; + muted?: boolean; + relayUrl: string; + threadId: string; + voice: string; +}; + +export type CodexVoiceTargetInput = Omit & { + voice?: string; +}; + +export type CodexVoiceSessionState = { + error: string | null; + muted: boolean; + phase: "starting" | "listening" | "ending" | "error"; + transcript: string | null; +}; + +export type VoiceRoomTranscriptEntry = { + id: number; + speakerName: string; + speakerType: "agent" | "human"; + text: string; + timestamp: number; +}; + +export type VoiceRoomDirectedTurn = { + id: number; + recipientThreadId: string; + text: string; +}; + +export type VoiceRoomSpeakerLease = { + threadId: string; + turnId: number; +}; + +export const VOICE_ROOM_PALETTE = [ + "sol", + "cove", + "ember", + "breeze", + "arbor", + "vale", + "juniper", + "maple", + "spruce", +] as const; + +const ACTIVE_STORAGE_KEY = "buzz.voice-room.active.v1"; +const SAVED_STORAGE_KEY = "buzz.voice-room.saved.v1"; + +function readTargets(key: string): CodexVoiceTarget[] { + if (typeof localStorage === "undefined") return []; + try { + const parsed = JSON.parse(localStorage.getItem(key) ?? "[]"); + return Array.isArray(parsed) ? parsed.filter(isPersistedVoiceTarget) : []; + } catch { + return []; + } +} + +function isPersistedVoiceTarget(value: unknown): value is CodexVoiceTarget { + if (!value || typeof value !== "object") return false; + const target = value as Partial; + return ( + typeof target.agentName === "string" && + typeof target.agentPubkey === "string" && + typeof target.channelId === "string" && + (target.mode === "native" || target.mode === "proxy") && + typeof target.relayUrl === "string" && + typeof target.threadId === "string" && + typeof target.voice === "string" + ); +} + +function persistTargets(key: string, targets: readonly CodexVoiceTarget[]) { + if (typeof localStorage === "undefined") return; + try { + localStorage.setItem(key, JSON.stringify(targets)); + } catch { + // Voice remains usable when storage is unavailable. + } +} + +export function chooseAvailableVoice( + targets: readonly CodexVoiceTarget[], +): string { + const assigned = new Set(targets.map((target) => target.voice)); + return ( + VOICE_ROOM_PALETTE.find((voice) => !assigned.has(voice)) ?? + VOICE_ROOM_PALETTE[targets.length % VOICE_ROOM_PALETTE.length] + ); +} + +export function addVoiceTarget( + targets: readonly CodexVoiceTarget[], + target: CodexVoiceTarget, +): CodexVoiceTarget[] { + if (targets.some((current) => current.threadId === target.threadId)) { + return [...targets]; + } + return [...targets, target]; +} + +export function removeVoiceTarget( + targets: readonly CodexVoiceTarget[], + threadId: string, +): CodexVoiceTarget[] { + return targets.filter((target) => target.threadId !== threadId); +} + +export function hasVoiceTarget( + targets: readonly CodexVoiceTarget[], + threadId: string, +): boolean { + return targets.some((target) => target.threadId === threadId); +} + +let activeTargets: CodexVoiceTarget[] = readTargets(ACTIVE_STORAGE_KEY); +let savedTargets: CodexVoiceTarget[] = readTargets(SAVED_STORAGE_KEY); +let sessionStates: Record = {}; +let roomTranscript: VoiceRoomTranscriptEntry[] = []; +let roomTranscriptSequence = 0; +let roomOutputMuted = false; +let directedTurns: VoiceRoomDirectedTurn[] = []; +let directedTurnSequence = 0; +let speakerLease: VoiceRoomSpeakerLease | null = null; +let speakerLeaseTimer: ReturnType | null = null; +const listeners = new Set<() => void>(); + +function emitChange() { + for (const listener of listeners) listener(); +} + +function replaceActiveTargets(next: CodexVoiceTarget[]) { + activeTargets = next; + persistTargets(ACTIVE_STORAGE_KEY, next); + emitChange(); +} + +function rememberTarget(target: CodexVoiceTarget) { + savedTargets = [ + target, + ...savedTargets.filter( + (saved) => + saved.agentPubkey.toLowerCase() !== target.agentPubkey.toLowerCase(), + ), + ]; + persistTargets(SAVED_STORAGE_KEY, savedTargets); +} + +export function startVoiceTarget(target: CodexVoiceTargetInput) { + const saved = savedTargets.find( + (current) => + current.agentPubkey.toLowerCase() === target.agentPubkey.toLowerCase(), + ); + const completeTarget: CodexVoiceTarget = { + ...target, + muted: target.muted ?? saved?.muted ?? false, + voice: target.voice ?? saved?.voice ?? chooseAvailableVoice(activeTargets), + }; + const next = addVoiceTarget(activeTargets, completeTarget); + rememberTarget(completeTarget); + if (next.length === activeTargets.length) return; + replaceActiveTargets(next); +} + +export function saveVoiceTargetPreference(target: CodexVoiceTargetInput) { + const saved = savedTargets.find( + (current) => + current.agentPubkey.toLowerCase() === target.agentPubkey.toLowerCase(), + ); + const completeTarget: CodexVoiceTarget = { + ...target, + muted: target.muted ?? saved?.muted ?? false, + voice: target.voice ?? saved?.voice ?? chooseAvailableVoice(activeTargets), + }; + rememberTarget(completeTarget); + emitChange(); +} + +export function endVoiceTarget(threadId: string) { + const next = removeVoiceTarget(activeTargets, threadId); + if (next.length === activeTargets.length) return; + const { [threadId]: _removed, ...remainingStates } = sessionStates; + sessionStates = remainingStates; + replaceActiveTargets(next); +} + +export function setVoiceTargetMuted(threadId: string, muted: boolean) { + const next = activeTargets.map((target) => + target.threadId === threadId ? { ...target, muted } : target, + ); + if (next.every((target, index) => target === activeTargets[index])) return; + const changed = next.find((target) => target.threadId === threadId); + if (changed) rememberTarget(changed); + replaceActiveTargets(next); +} + +export function setVoiceTargetVoice(threadId: string, voice: string) { + if ( + !VOICE_ROOM_PALETTE.includes(voice as (typeof VOICE_ROOM_PALETTE)[number]) + ) { + return; + } + const next = activeTargets.map((target) => + target.threadId === threadId ? { ...target, voice } : target, + ); + if (next.every((target, index) => target === activeTargets[index])) return; + const changed = next.find((target) => target.threadId === threadId); + if (changed) rememberTarget(changed); + replaceActiveTargets(next); +} + +export function updateVoiceSessionState( + threadId: string, + update: Partial, +) { + const current = sessionStates[threadId] ?? { + error: null, + muted: false, + phase: "starting", + transcript: null, + }; + sessionStates = { + ...sessionStates, + [threadId]: { ...current, ...update }, + }; + emitChange(); +} + +export function appendVoiceRoomTranscript(input: { + speakerName: string; + speakerType: "agent" | "human"; + text: string; +}) { + const text = input.text.trim(); + if (!text) return; + const timestamp = Date.now(); + const duplicateWindowMs = input.speakerType === "human" ? 3_000 : 1_000; + const duplicate = [...roomTranscript] + .reverse() + .find( + (entry) => + timestamp - entry.timestamp <= duplicateWindowMs && + entry.speakerType === input.speakerType && + entry.speakerName === input.speakerName && + entry.text === text, + ); + if (duplicate) return; + roomTranscriptSequence += 1; + roomTranscript = [ + ...roomTranscript.slice(-199), + { ...input, id: roomTranscriptSequence, text, timestamp }, + ]; + emitChange(); +} + +export function setVoiceRoomOutputMuted(muted: boolean) { + if (roomOutputMuted === muted) return; + roomOutputMuted = muted; + emitChange(); +} + +export function routeVoiceRoomTurn(text: string): VoiceRoomDirectedTurn | null { + const recipient = resolveVoiceTurnRecipient(text, activeTargets); + if (!recipient) return null; + directedTurnSequence += 1; + const turn = { + id: directedTurnSequence, + recipientThreadId: recipient.threadId, + text: text.trim(), + }; + directedTurns = [...directedTurns.slice(-49), turn]; + claimVoiceRoomSpeaker(recipient.threadId, turn.id); + emitChange(); + return turn; +} + +export function claimVoiceRoomSpeaker(threadId: string, turnId: number) { + if (speakerLeaseTimer) clearTimeout(speakerLeaseTimer); + speakerLease = { threadId, turnId }; + speakerLeaseTimer = setTimeout(() => { + if (speakerLease?.turnId !== turnId) return; + speakerLease = null; + speakerLeaseTimer = null; + emitChange(); + }, 45_000); + emitChange(); +} + +export function releaseVoiceRoomSpeaker(threadId: string) { + if (speakerLease?.threadId !== threadId) return; + if (speakerLeaseTimer) clearTimeout(speakerLeaseTimer); + speakerLeaseTimer = null; + speakerLease = null; + emitChange(); +} + +export function getVoiceRoomSnapshot() { + return { + activeTargets, + outputMuted: roomOutputMuted, + directedTurns, + speakerLease, + } as const; +} + +export function useVoiceRoomDirectedTurns(): readonly VoiceRoomDirectedTurn[] { + return React.useSyncExternalStore( + subscribe, + () => directedTurns, + () => directedTurns, + ); +} + +export function useVoiceRoomSpeakerLease(): VoiceRoomSpeakerLease | null { + return React.useSyncExternalStore( + subscribe, + () => speakerLease, + () => speakerLease, + ); +} + +function subscribe(listener: () => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function useCodexVoiceTargets(): readonly CodexVoiceTarget[] { + return React.useSyncExternalStore( + subscribe, + () => activeTargets, + () => activeTargets, + ); +} + +export function useSavedCodexVoiceTargets(): readonly CodexVoiceTarget[] { + return React.useSyncExternalStore( + subscribe, + () => savedTargets, + () => savedTargets, + ); +} + +export function useCodexVoiceSessionStates(): Readonly< + Record +> { + return React.useSyncExternalStore( + subscribe, + () => sessionStates, + () => sessionStates, + ); +} + +export function useVoiceRoomTranscript(): readonly VoiceRoomTranscriptEntry[] { + return React.useSyncExternalStore( + subscribe, + () => roomTranscript, + () => roomTranscript, + ); +} + +export function useVoiceRoomOutputMuted(): boolean { + return React.useSyncExternalStore( + subscribe, + () => roomOutputMuted, + () => roomOutputMuted, + ); +} diff --git a/desktop/src/features/agents/voiceTurnRouting.test.mjs b/desktop/src/features/agents/voiceTurnRouting.test.mjs new file mode 100644 index 0000000000..bd7c5bbc22 --- /dev/null +++ b/desktop/src/features/agents/voiceTurnRouting.test.mjs @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + resolveVoiceTurnRecipient, + shouldForwardVoiceTurn, +} from "./voiceTurnRouting.ts"; + +test("forwards every room turn to Orchestrator", () => { + assert.equal( + shouldForwardVoiceTurn("Please review this", "Orchestrator"), + true, + ); +}); + +test("keeps an unaddressed human turn away from a specialist", () => { + assert.equal(shouldForwardVoiceTurn("Please review this", "Builder"), false); +}); + +test("forwards a turn when the specialist is explicitly named", () => { + assert.equal( + shouldForwardVoiceTurn( + "Builder, please implement the approved plan", + "Builder", + ), + true, + ); +}); + +test("forwards natural questions and delegated requests", () => { + assert.equal( + shouldForwardVoiceTurn("What does Architect think?", "Architect"), + true, + ); + assert.equal( + shouldForwardVoiceTurn("Could Builder implement this?", "Builder"), + true, + ); + assert.equal(shouldForwardVoiceTurn("Tell Builder to stop", "Builder"), true); + assert.equal( + shouldForwardVoiceTurn("Can you invite Researcher now?", "Researcher"), + true, + ); +}); + +test("does not wake specialists for narration or quoted examples", () => { + assert.equal( + shouldForwardVoiceTurn( + "Orchestrator, Builder, and Explorer are all listening", + "Builder", + ), + false, + ); + assert.equal( + shouldForwardVoiceTurn( + "So Builder join and Builder stop should change participation immediately", + "Builder", + ), + false, + ); + assert.equal( + shouldForwardVoiceTurn( + "Builder is the primary implementation agent", + "Builder", + ), + false, + ); +}); + +test("does not match a specialist name inside another word", () => { + assert.equal( + shouldForwardVoiceTurn("Use the form builder", "Builder Pro"), + false, + ); +}); + +const room = [ + { agentName: "Orchestrator", threadId: "orchestrator" }, + { agentName: "Builder", threadId: "builder" }, + { agentName: "Architect", threadId: "architect" }, +]; + +test("routes an unaddressed turn only to Orchestrator", () => { + assert.equal( + resolveVoiceTurnRecipient("Please review this", room)?.threadId, + "orchestrator", + ); +}); + +test("routes an explicitly addressed turn to one specialist", () => { + assert.equal( + resolveVoiceTurnRecipient("Architect, what do you think?", room)?.threadId, + "architect", + ); +}); + +test("routes a name-led concise request from live speech to the specialist", () => { + assert.equal( + resolveVoiceTurnRecipient( + "Architect in one sentence is programmatic voice room join end to end ready and why", + room, + )?.threadId, + "architect", + ); +}); diff --git a/desktop/src/features/agents/voiceTurnRouting.ts b/desktop/src/features/agents/voiceTurnRouting.ts new file mode 100644 index 0000000000..f4fd2516fa --- /dev/null +++ b/desktop/src/features/agents/voiceTurnRouting.ts @@ -0,0 +1,69 @@ +function normalizeVoiceText(value: string): string { + return value + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +export function shouldForwardVoiceTurn( + transcript: string, + agentName: string, +): boolean { + const normalizedAgent = normalizeVoiceText(agentName); + if (!normalizedAgent) return false; + if (normalizedAgent === "orchestrator") return true; + + const normalizedTranscript = normalizeVoiceText(transcript); + if (!normalizedTranscript) return false; + + const agent = escapeRegExp(normalizedAgent); + const conciseQualifier = + "(?:briefly|quickly|in (?:a|one) sentence|in (?:a few|one) words?)"; + const directAddress = new RegExp( + `^(?:(?:hey|okay|ok|please) )?${agent}(?:$| (?:(?:please )?(?:ask|tell|have|let|bring|invite|remove|start|stop|mute|unmute|join|leave|implement|review|check|explain|answer|help|handle|take|look)|${conciseQualifier}|(?:what|how|why|when|where|can|could|would|should|will|do))(?: |$))`, + ); + const directRequest = new RegExp( + `^(?:please )?(?:ask|tell|have|let|bring|invite|remove|start|stop|mute|unmute) ${agent}(?: |$)`, + ); + const directQuestion = new RegExp( + `^(?:(?:what|how|why|when|where) (?:does|would|should|can|could|is)|(?:can|could|would|should|will|is)) ${agent}(?: |$)`, + ); + const delegatedRequest = new RegExp( + `^(?:can|could|would|will) (?:you |we )?(?:ask|tell|have|let|bring|invite|remove|start|stop|mute|unmute) ${agent}(?: |$)`, + ); + + return ( + directAddress.test(normalizedTranscript) || + directRequest.test(normalizedTranscript) || + directQuestion.test(normalizedTranscript) || + delegatedRequest.test(normalizedTranscript) + ); +} + +export function resolveVoiceTurnRecipient< + T extends { + agentName: string; + threadId: string; + }, +>(transcript: string, targets: readonly T[]): T | null { + const specialist = targets.find( + (target) => + normalizeVoiceText(target.agentName) !== "orchestrator" && + shouldForwardVoiceTurn(transcript, target.agentName), + ); + if (specialist) return specialist; + return ( + targets.find( + (target) => normalizeVoiceText(target.agentName) === "orchestrator", + ) ?? + targets[0] ?? + null + ); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/desktop/src/features/channels/ui/AgentSessionThreadPanel.tsx b/desktop/src/features/channels/ui/AgentSessionThreadPanel.tsx index 641b81490b..813a0d88a5 100644 --- a/desktop/src/features/channels/ui/AgentSessionThreadPanel.tsx +++ b/desktop/src/features/channels/ui/AgentSessionThreadPanel.tsx @@ -11,6 +11,7 @@ import { toast } from "sonner"; import { useAgentWorking } from "@/features/agents/agentWorkingSignal"; import { isManagedAgentActive } from "@/features/agents/lib/managedAgentControlActions"; import { + deriveLatestSessionId, mergeObserverEventWindows, observerEventScrollId, scopeByChannel, @@ -18,6 +19,7 @@ import { import { deriveTranscriptBlockIds } from "@/features/agents/ui/agentSessionTranscriptGrouping"; import type { ObserverEvent } from "@/features/agents/ui/agentSessionTypes"; import { ManagedAgentSessionPanel } from "@/features/agents/ui/ManagedAgentSessionPanel"; +import { CodexVoiceLauncher } from "@/features/agents/ui/CodexVoiceController"; import { useArchivedChannelEvents, useObserverEvents, @@ -25,6 +27,10 @@ import { import { useAnchoredScroll } from "@/features/messages/ui/useAnchoredScroll"; import { useStableArrayShallow } from "@/shared/hooks/useStableReference"; import { cancelManagedAgentTurn } from "@/shared/api/agentControl"; +import { + getCodexVoiceLink, + rememberCodexVoiceLink, +} from "@/shared/api/codexVoice"; import type { Channel } from "@/shared/api/types"; import { useEscapeKey } from "@/shared/hooks/useEscapeKey"; import { useIsThreadPanelOverlay } from "@/shared/hooks/use-mobile"; @@ -132,6 +138,42 @@ export function AgentSessionThreadPanel({ () => getLatestActivityTimestamp(combinedHeaderEvents), [combinedHeaderEvents], ); + const latestSessionId = React.useMemo( + () => deriveLatestSessionId(combinedHeaderEvents), + [combinedHeaderEvents], + ); + const [rememberedVoiceThreadId, setRememberedVoiceThreadId] = React.useState< + string | null + >(null); + React.useEffect(() => { + let active = true; + setRememberedVoiceThreadId(null); + if (!sessionChannelId) { + return () => { + active = false; + }; + } + void getCodexVoiceLink(agent.pubkey, sessionChannelId).then((threadId) => { + if (active) { + setRememberedVoiceThreadId(threadId); + } + }); + return () => { + active = false; + }; + }, [agent.pubkey, sessionChannelId]); + React.useEffect(() => { + if (!latestSessionId || !sessionChannelId) { + return; + } + setRememberedVoiceThreadId(latestSessionId); + void rememberCodexVoiceLink( + agent.pubkey, + sessionChannelId, + latestSessionId, + ); + }, [agent.pubkey, latestSessionId, sessionChannelId]); + const voiceThreadId = latestSessionId ?? rememberedVoiceThreadId; const lastUpdatedLabel = formatLastUpdatedLabel(latestActivityAt, now); const lastUpdatedTitle = latestActivityAt === null @@ -483,6 +525,18 @@ export function AgentSessionThreadPanel({ {agentHeaderContent} } + footer={ + agent.agentSource === "managed" ? ( + + ) : null + } > setIsMembersSidebarOpen((prev) => !prev), [], ); - - const channelHeader = React.useMemo( - () => ( - - ), - [ - activeChannel, - activeChannelEphemeralDisplay, - activeChannelTitle, - shouldCompactHeaderActions, - activeDmAvatarUrl, + const dmVoiceAgent = React.useMemo(() => { + return resolveDmCodexVoiceAgent( + activeChannel?.channelType, activeDmHeaderParticipants, - activeDmPresenceStatus, - channelHeaderChromeRef, - currentPubkey, - isAddBotOpen, - joinChannelMutation.isPending, - joinChannelMutation.mutateAsync, - handleManageChannel, - handleToggleMembers, - isSinglePanelView, - ], + managedAgents, + ); + }, [activeChannel?.channelType, activeDmHeaderParticipants, managedAgents]); + const handleOpenDmVoice = React.useCallback( + (pubkey: string) => handleOpenAgentSession(pubkey, activeChannel?.id), + [activeChannel?.id, handleOpenAgentSession], + ); + + const channelHeader = ( + ); return ( diff --git a/desktop/src/features/channels/ui/ChannelScreenHeader.tsx b/desktop/src/features/channels/ui/ChannelScreenHeader.tsx index 4c545baf68..1e4f3422b7 100644 --- a/desktop/src/features/channels/ui/ChannelScreenHeader.tsx +++ b/desktop/src/features/channels/ui/ChannelScreenHeader.tsx @@ -1,4 +1,4 @@ -import { LogIn } from "lucide-react"; +import { LogIn, Mic } from "lucide-react"; import type * as React from "react"; import { ChatHeader } from "@/features/chat/ui/ChatHeader"; @@ -36,10 +36,12 @@ type ChannelScreenHeaderProps = { currentPubkey?: string; isAddBotOpen?: boolean; isJoining?: boolean; + dmVoiceAgent?: { name: string; pubkey: string } | null; showHeaderContent?: boolean; transparentChrome?: boolean; onAddBotOpenChange?: (open: boolean) => void; onJoinChannel?: () => Promise; + onOpenDmVoice?: (pubkey: string) => void; onManageChannel: () => void; onToggleMembers: () => void; }; @@ -56,10 +58,12 @@ export function ChannelScreenHeader({ currentPubkey, isAddBotOpen, isJoining = false, + dmVoiceAgent, onAddBotOpenChange, showHeaderContent = true, transparentChrome = false, onJoinChannel, + onOpenDmVoice, onManageChannel, onToggleMembers, }: ChannelScreenHeaderProps) { @@ -86,15 +90,33 @@ export function ChannelScreenHeader({ {isJoining ? "Joining…" : "Join"} ) : ( - +
+ {dmVoiceAgent && onOpenDmVoice ? ( + + ) : null} + +
) ) : null; diff --git a/desktop/src/features/channels/ui/dmCodexVoiceAgent.ts b/desktop/src/features/channels/ui/dmCodexVoiceAgent.ts new file mode 100644 index 0000000000..844fc81ea2 --- /dev/null +++ b/desktop/src/features/channels/ui/dmCodexVoiceAgent.ts @@ -0,0 +1,20 @@ +import type { ManagedAgent } from "@/shared/api/types"; +import { normalizePubkey } from "@/shared/lib/pubkey"; + +type DmParticipant = { + pubkey: string; +}; + +export function resolveDmCodexVoiceAgent( + channelType: string | undefined, + participants: DmParticipant[], + managedAgents: ManagedAgent[], +): { name: string; pubkey: string } | null { + if (channelType !== "dm" || participants.length !== 1) return null; + const participantPubkey = normalizePubkey(participants[0].pubkey); + const agent = managedAgents.find( + (candidate) => normalizePubkey(candidate.pubkey) === participantPubkey, + ); + if (!agent) return null; + return { name: agent.name, pubkey: agent.pubkey }; +} diff --git a/desktop/src/features/channels/ui/useChannelAgentSessions.ts b/desktop/src/features/channels/ui/useChannelAgentSessions.ts index 8420c37327..737c3a9bac 100644 --- a/desktop/src/features/channels/ui/useChannelAgentSessions.ts +++ b/desktop/src/features/channels/ui/useChannelAgentSessions.ts @@ -21,6 +21,7 @@ export type ChannelAgentSessionAgent = Pick< > & { agentSource: "managed" | "member-bot" | "relay"; canInterruptTurn: boolean; + relayUrl?: string; channelIds?: string[]; channels?: string[]; }; @@ -67,6 +68,7 @@ export function buildChannelAgentSessionCandidates({ pubkey: agent.pubkey, name: agent.name, status: relayStatusToManagedStatus(agent.status), + relayUrl: "", agentSource: "relay", canInterruptTurn: false, channelIds: agent.channelIds, @@ -81,6 +83,7 @@ export function buildChannelAgentSessionCandidates({ pubkey: agent.pubkey, name: agent.name, status: agent.status, + relayUrl: agent.relayUrl, agentSource: "managed", canInterruptTurn: true, channelIds: existing?.channelIds, @@ -98,6 +101,7 @@ export function buildChannelAgentSessionCandidates({ pubkey: member.pubkey, name: member.displayName ?? truncatePubkey(member.pubkey), status: "deployed", + relayUrl: "", agentSource: "member-bot", canInterruptTurn: false, }); @@ -219,6 +223,7 @@ export function useChannelAgentSessions({ setThreadScrollTargetId(null); setThreadReplyTargetId(null); setChannelManagementOpen(false); + setProfilePanelPubkey(null); setOpenAgentSessionPubkey(pubkey); // Fall back to activeChannelId so opening from within a channel always // scopes the panel to that channel — even when no explicit channelId is @@ -237,6 +242,7 @@ export function useChannelAgentSessions({ setOpenAgentSessionChannelId, setOpenAgentSessionPubkey, setOpenThreadHeadId, + setProfilePanelPubkey, setThreadReplyTargetId, setThreadScrollTargetId, ], diff --git a/desktop/src/features/profile/ui/UserProfilePanel.tsx b/desktop/src/features/profile/ui/UserProfilePanel.tsx index cb188dd008..4c1f8663db 100644 --- a/desktop/src/features/profile/ui/UserProfilePanel.tsx +++ b/desktop/src/features/profile/ui/UserProfilePanel.tsx @@ -341,7 +341,7 @@ export function UserProfilePanel({ const canViewActivity = viewerIsOwner && Boolean(effectivePubkey) && - canOpenAgentActivity(effectivePubkey); + (managedAgent !== undefined || canOpenAgentActivity(effectivePubkey)); const canOpenAgentLogs = isOwner === true && managedAgent?.backend.type === "local"; const canInstantiateAgent = diff --git a/desktop/src/features/sidebar/ui/AppSidebar.tsx b/desktop/src/features/sidebar/ui/AppSidebar.tsx index 55f467f215..a7a99902bd 100644 --- a/desktop/src/features/sidebar/ui/AppSidebar.tsx +++ b/desktop/src/features/sidebar/ui/AppSidebar.tsx @@ -103,6 +103,7 @@ type AppSidebarProps = { | "channel" | "messages" | "agents" + | "voice" | "workflows" | "pulse" | "projects"; @@ -145,6 +146,7 @@ type AppSidebarProps = { onSelectAgents: () => void; onSelectProjects: () => void; onSelectPulse: () => void; + onSelectVoice: () => void; onSelectWorkflows: () => void; onSelectHome: () => void; onSelectChannel: (channelId: string) => void; @@ -214,6 +216,7 @@ export function AppSidebar({ onSelectAgents, onSelectProjects, onSelectPulse, + onSelectVoice, onSelectWorkflows, onSelectHome, onSelectChannel, @@ -612,6 +615,7 @@ export function AppSidebar({ onSelectHome={onSelectHome} onSelectProjects={onSelectProjects} onSelectPulse={onSelectPulse} + onSelectVoice={onSelectVoice} onSelectWorkflows={onSelectWorkflows} selectedView={selectedView} /> diff --git a/desktop/src/features/sidebar/ui/AppSidebarPinnedHeader.tsx b/desktop/src/features/sidebar/ui/AppSidebarPinnedHeader.tsx index a673492ef1..21e6ecc4ea 100644 --- a/desktop/src/features/sidebar/ui/AppSidebarPinnedHeader.tsx +++ b/desktop/src/features/sidebar/ui/AppSidebarPinnedHeader.tsx @@ -1,4 +1,11 @@ -import { Activity, Bot, FolderGit2, Inbox, Zap } from "lucide-react"; +import { + Activity, + AudioLines, + Bot, + FolderGit2, + Inbox, + Zap, +} from "lucide-react"; import { TopbarSearch } from "@/features/search/ui/TopbarSearch"; import { FeatureGate } from "@/shared/features"; @@ -17,6 +24,7 @@ type SidebarSelectedView = | "channel" | "messages" | "agents" + | "voice" | "workflows" | "pulse" | "projects"; @@ -41,6 +49,7 @@ type AppSidebarPrimaryMenuProps = { onSelectHome: () => void; onSelectProjects: () => void; onSelectPulse: () => void; + onSelectVoice: () => void; onSelectWorkflows: () => void; selectedView: SidebarSelectedView; }; @@ -86,6 +95,7 @@ export function AppSidebarPrimaryMenu({ onSelectHome, onSelectProjects, onSelectPulse, + onSelectVoice, onSelectWorkflows, selectedView, }: AppSidebarPrimaryMenuProps) { @@ -155,6 +165,18 @@ export function AppSidebarPrimaryMenu({ Agents + + + + Voice + + ; +}; + +export type CodexVoiceEvent = { + method: string; + params: { + threadId?: string; + sdp?: string; + role?: string; + delta?: string; + text?: string; + message?: string; + reason?: string | null; + }; +}; + +let voiceLinkRevision = 0; +const voiceLinkListeners = new Set<() => void>(); + +export function subscribeCodexVoiceLinkChanges(listener: () => void) { + voiceLinkListeners.add(listener); + return () => { + voiceLinkListeners.delete(listener); + }; +} + +export function getCodexVoiceLinkRevision() { + return voiceLinkRevision; +} + +function notifyCodexVoiceLinkChange() { + voiceLinkRevision += 1; + for (const listener of voiceLinkListeners) listener(); +} + +export function getCodexVoiceCapability( + pubkey: string, + relayUrl: string, +): Promise { + return invokeTauri("get_codex_voice_capability", { pubkey, relayUrl }); +} + +export function requestMicrophoneAccess(): Promise { + return invokeTauri("request_microphone_access"); +} + +export function getCodexVoiceStatus(): Promise { + return invokeTauri("get_codex_voice_status"); +} + +export function getCodexVoiceLink( + pubkey: string, + channelId: string, +): Promise { + return invokeTauri("get_codex_voice_link", { pubkey, channelId }); +} + +export function getCodexVoiceTargetLink( + pubkey: string, +): Promise { + return invokeTauri("get_codex_voice_target_link", { pubkey }); +} + +export async function rememberCodexVoiceLink( + pubkey: string, + channelId: string, + threadId: string, +): Promise { + await invokeTauri("remember_codex_voice_link", { + pubkey, + channelId, + threadId, + }); + notifyCodexVoiceLinkChange(); +} + +export function startCodexVoice(input: { + threadId: string; + pubkey: string; + agentName: string; + relayUrl: string; + voice: string; + sdp: string; +}): Promise { + return invokeTauri("start_codex_voice", input); +} + +export function speakCodexVoice(threadId: string, text: string): Promise { + return invokeTauri("speak_codex_voice", { threadId, text }); +} + +export function stopCodexVoice(threadId: string): Promise { + return invokeTauri("stop_codex_voice", { threadId }); +} + +export function setCodexVoiceMuted( + threadId: string, + muted: boolean, +): Promise { + return invokeTauri("set_codex_voice_muted", { threadId, muted }); +}