diff --git a/.gitignore b/.gitignore index c1cca80..f6c3cb8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ build/ develop-eggs/ dist/ downloads/ +frontend/node_modules/ eggs/ .eggs/ lib/ diff --git a/backend/api/ws_voice.py b/backend/api/ws_voice.py index 92e1fea..6cddd07 100644 --- a/backend/api/ws_voice.py +++ b/backend/api/ws_voice.py @@ -55,7 +55,8 @@ async def client_to_openai() -> None: logger.error("client_to_openai_error", error=str(e)) async def on_openai_event(event_type: str, payload: dict[str, Any]) -> None: - if event_type == "response.audio.delta": + # GA Event for audio streaming + if event_type == "response.output_audio.delta": delta = payload.get("delta") if delta: await websocket.send_bytes(base64.b64decode(delta)) @@ -86,7 +87,7 @@ async def on_openai_event(event_type: str, payload: dict[str, Any]) -> None: with contextlib.suppress(Exception): await websocket.close(code=1000) - elif event_type == "response.audio_transcript.done": + elif event_type == "response.output_transcript.done": text = payload.get("transcript") if text: transcript.append(f"AI: {text}") diff --git a/backend/services/openai_realtime.py b/backend/services/openai_realtime.py index 0d4900f..88cc561 100644 --- a/backend/services/openai_realtime.py +++ b/backend/services/openai_realtime.py @@ -26,8 +26,8 @@ def __init__(self, settings: Settings) -> None: async def connect(self, session_id: str, client_ws: Any) -> None: url = f"wss://api.openai.com/v1/realtime?model={self.model}" headers = { - "Authorization": f"Bearer {self.api_key}", - "OpenAI-Beta": "realtime=v1", + "Authorization": f"Bearer {self.api_key}" + # OpenAI-Beta header strictly removed for GA compatibility } session = VoiceSession(id=session_id, ws_client=client_ws, status=SessionStatus.CONNECTING) @@ -40,6 +40,7 @@ async def connect(self, session_id: str, client_ws: Any) -> None: update_event = { "type": "session.update", "session": { + "type": "realtime", "instructions": "You are a helpful restaurant booking agent. Be concise.", "tools": [ { @@ -54,8 +55,10 @@ async def connect(self, session_id: str, client_ws: Any) -> None: }, ], "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", + "audio": { + "input": {"format": {"type": "audio/pcm", "rate": 24000}}, + "output": {"format": {"type": "audio/pcm", "rate": 24000}}, + }, "turn_detection": {"type": "server_vad"}, }, } diff --git a/frontend/src/hooks/useVoiceSession.js b/frontend/src/hooks/useVoiceSession.js index 0ddc598..e4d2512 100644 --- a/frontend/src/hooks/useVoiceSession.js +++ b/frontend/src/hooks/useVoiceSession.js @@ -12,7 +12,13 @@ export function useVoiceSession() { const startSession = useCallback(async () => { try { setStatus("PROCESSING"); - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); streamRef.current = stream; const ws = new WebSocket(getWsUrl("/ws/voice")); diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 80d6550..bc142fb 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -7,11 +7,11 @@ export default defineConfig({ port: 3000, proxy: { "/api": { - target: "http://127.0.0.1:8000", + target: "http://127.0.0.1:8080", changeOrigin: true, }, "/ws": { - target: "ws://127.0.0.1:8000", + target: "ws://127.0.0.1:8080", ws: true, }, },