From f711e7ac9672d7964545a6a4dc2db36c1e6ea489 Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Thu, 13 Aug 2026 08:45:53 -0700
Subject: [PATCH 1/9] =?UTF-8?q?Update=20audioStartTime=20and=20audioEndTim?=
 =?UTF-8?q?e=20to=20be=20relative=20to=20start=20of=20aud=E2=80=A6=20(#203?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update audioStartTime and audioEndTime to be relative to start of audio stream

We want to switch to  stream-Relative (0-based) impl. since:
- In Web Audio and Media APIs (HTMLMediaElement.currentTime, AudioContext.currentTime, WebCodecs VideoFrame.timestamp), media timelines are always 0-based offsets relative to stream start, not the time origin.
- Immune to inter-process jitter since SODA and audio capture run in a separate utility/browser process. Translating stream offsets to the renderer's timeOrigin relies on estimating when IPC AudioStarted() arrived, which introduces IPC latency jitter. Stream-relative offsets are not prune to this and aligns with the raw audio frames.
- If SpeechRecognition is used with a pre-recorded MediaStreamTrack, a 0-based stream offset reflects the actual position in the audio track regardless of when the webpage was loaded.
- Being relative to performance.timeOrigin doesn't make sense in general for the Web Speech API because it assumes that the audio source is live. Since a SpeechRecognizer can also be created for a prerecorded media stream I think the timestamps on the speech recognition events should be relative to the position in that media stream.

See https://crbug.com/542330168 for more details.

* Update speech recognition explainer with timestamp conversion

Added a section on converting stream timestamps to document time origin and provided a live translation latency example with code.

* Minor update to close live transcription latency measurement example

Added separator before security section and closed example section for live transcription measurements.

* respond to review comment
---
 .../speech-recognition-result-timestamps.md   | 62 ++++++++++++++++++-
 1 file changed, 59 insertions(+), 3 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index c6ccd3b..0200cc5 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -17,10 +17,10 @@ We propose extending the `SpeechRecognitionResult` interface to include optional
 
 ```webidl
 partial interface SpeechRecognitionResult {
-    // Start timestamp of the audio segment in milliseconds (relative to time origin)
+    // Start timestamp of the audio segment in milliseconds (relative to the start of the audio stream)
     readonly attribute DOMHighResTimeStamp? audioStartTime;
 
-    // End timestamp of the audio segment in milliseconds (relative to time origin)
+    // End timestamp of the audio segment in milliseconds (relative to the start of the audio stream)
     readonly attribute DOMHighResTimeStamp? audioEndTime;
 };
 ```
@@ -39,7 +39,7 @@ recognition.interimResults = true;
 recognition.onresult = (event) => {
   const result = event.results[event.resultIndex];
 
-  if (result.audioEndTime !== null) {
+  if (result.audioEndTime !== null && result.audioEndTime !== undefined) {
     // Calculate on-device processing latency
     const processingLatencyMs = event.timeStamp - result.audioEndTime;
 
@@ -54,6 +54,62 @@ recognition.onresult = (event) => {
 recognition.start();
 ```
 
+## Converting Stream Timestamps to Document Time Origin
+
+`audioStartTime` and `audioEndTime` are defined as media-local offsets in milliseconds relative to the start of the audio stream ($t = 0.0\text{ms}$).
+
+For real-time applications such as **live translation**, **subtitling overlays**, and **audio-visual sync**, developers often need to map these stream offsets to the document's global timeline (`DOMHighResTimeStamp` / `performance.now()`).
+
+### Pattern: Capturing the Audio Timeline Origin
+
+To convert stream-relative timestamps to document time coordinates:
+1. Record the baseline timestamp when the `audiostart` event fires (`event.timeStamp` is a `DOMHighResTimeStamp` relative to `timeOrigin`).
+2. Add the result's `audioStartTime` and `audioEndTime` offsets to that baseline.
+
+$$\text{absoluteStartTime} = \text{audioOrigin} + \text{result.audioStartTime}$$
+$$\text{absoluteEndTime} = \text{audioOrigin} + \text{result.audioEndTime}$$
+
+### Measuring Live Translation Latency Example
+
+In live speech translation workflows, measuring both **Speech-to-Text (STT) latency** and **Machine Translation (MT) end-to-end latency** is essential:
+
+```javascript
+const recognition = new SpeechRecognition();
+recognition.continuous = true;
+recognition.interimResults = true;
+
+let audioOriginTime = 0;
+
+// 1. Capture the audio stream's time origin on the document timeline
+recognition.onaudiostart = (event) => {
+  audioOriginTime = event.timeStamp;
+};
+
+recognition.onresult = async (event) => {
+  const result = event.results[event.resultIndex];
+  if (result.audioEndTime === null || result.audioEndTime === undefined) return;
+
+  // 2. Convert stream offsets to document time origin coordinates
+  const absoluteAudioStart = audioOriginTime + result.audioStartTime;
+  const absoluteAudioEnd = audioOriginTime + result.audioEndTime;
+
+  // 3. Compute ASR recognition latency
+  const asrLatencyMs = event.timeStamp - absoluteAudioEnd;
+
+  // 4. Perform live translation
+  const text = result[0].transcript;
+  const translationStartTime = performance.now();
+  const translatedText = await translateService.translate(text, 'es');
+  const translationEndTime = performance.now();
+
+  // 5. Total end-to-end latency from speaker utterance to translated subtitle
+  const totalE2ELatencyMs = translationEndTime - absoluteAudioEnd;
+
+  console.log(`ASR Processing Time: ${asrLatencyMs.toFixed(1)}ms`);
+  console.log(`Total Live Translation Delay: ${totalE2ELatencyMs.toFixed(1)}ms`);
+};
+```
+---
 ### Security and Privacy Considerations
 
 #### Fingerprinting Risk

From d5b1e6113e6373f4ea43623fa69262ddcc48aad5 Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Fri, 21 Aug 2026 15:33:52 -0700
Subject: [PATCH 2/9] Expand on limitations of existing speech recognition
 events in Alternatives and refactor Timestamps to use Seconds  (#205)

* Expand on limitations of existing speech recognition events

Added detailed explanations regarding the limitations of existing API surfaces for tracking latency in speech recognition in the Alternatives Considered section, including issues with `speechstart` and `speechend` events and the implications of modifying `event.timeStamp`.

* Refactor speech recognition timestamps to use seconds

Updated the speech recognition result timestamps to use seconds instead of milliseconds, improving consistency with other Web APIs. Added detailed explanations for the choice of time representation, proposed behavior, and security considerations. Based off of comments from https://github.com/WebAudio/web-speech-api/pull/205
---
 .../speech-recognition-result-timestamps.md   | 123 +++++++++++++-----
 1 file changed, 88 insertions(+), 35 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index 0200cc5..ff85d63 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -6,70 +6,103 @@
 
 The Web Speech API currently does not expose the start and end timestamps of the source audio corresponding to a given transcription result (`SpeechRecognitionResult`). This limitation creates two major challenges for API clients and end users:
 
-- **Timeline Association:** Developers cannot readily associate transcribed text with specific segments of the audio source, making it difficult to map generated captions to media timelines or audio tracks.
+- **Timeline Association:** Developers cannot readily associate transcribed text with specific segments of the audio source, making it difficult to map generated captions to media timelines, audio tracks, or video frames.
 - **Latency Tracking & Backend Failover:** With the adoption of on-device Automatic Speech Recognition (ASR) to improve privacy and reduce server costs, processing performance becomes heavily dependent on local client hardware resources. The Web Speech API acts as a "black box" regarding local processing delays. Developers cannot programmatically calculate transcription latency or detect when on-device models fall behind real-time. This leads to poor user experiences (e.g. caption lag during live video conferencing) and deprives applications of the signal needed to seamlessly fail over to high-performance cloud backends.
 
 ### Proposed Solution
 
-We propose extending the `SpeechRecognitionResult` interface to include optional (nullable) `audioStartTime` and `audioEndTime` attributes.
+We propose extending the `SpeechRecognitionResult` interface to include `audioStartTime` and `audioEndTime` attributes.
 
 #### Web IDL Definition
 
 ```webidl
 partial interface SpeechRecognitionResult {
-    // Start timestamp of the audio segment in milliseconds (relative to the start of the audio stream)
-    readonly attribute DOMHighResTimeStamp? audioStartTime;
+    // Start timestamp of the audio segment in seconds relative to the start of the audio stream (0.0s).
+    readonly attribute double audioStartTime;
 
-    // End timestamp of the audio segment in milliseconds (relative to the start of the audio stream)
-    readonly attribute DOMHighResTimeStamp? audioEndTime;
+    // End timestamp of the audio segment in seconds relative to the start of the audio stream.
+    readonly attribute double audioEndTime;
 };
 ```
 
+### Choice of Time Representation: Seconds as `double`
+
+The timestamps `audioStartTime` and `audioEndTime` are defined as `double` representing **seconds**, rather than `DOMHighResTimeStamp` (milliseconds). This design choice is based on the following considerations:
+
+1. **Consistency with Adjacent Web Audio & Media APIs:**
+   * In adjacent W3C media specifications, media-local stream timelines are universally represented in **seconds** as a `double`:
+     * **Web Audio API:** [`BaseAudioContext.currentTime`](https://webaudio.github.io/web-audio-api/#dom-baseaudiocontext-currenttime) (seconds)
+     * **HTML Media Elements:** [`HTMLMediaElement.currentTime`](https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime) (seconds)
+     * **AudioParam Scheduling:** [`AudioParam.setValueAtTime()`](https://webaudio.github.io/web-audio-api/#dom-audioparam-setvalueattime) (seconds)
+   * Using seconds ensures seamless interoperability when developers route audio between `<audio>`/`<video>` elements, Web Audio graphs, and `SpeechRecognition`, avoiding repetitive and error-prone unit conversions ($1000\times / \div 1000$).
+
+2. **Semantic Inaccuracy of `DOMHighResTimeStamp` for Media Streams:**
+   * Under the [W3C High Resolution Time Level 3](https://www.w3.org/TR/hr-time-3/#sec-domhighrestimestamp) specification, `DOMHighResTimeStamp` is strictly defined as a time coordinate in **milliseconds** measured relative to the global execution context's time origin (`performance.timeOrigin`).
+   * Because `audioStartTime` and `audioEndTime` represent a **media-local timeline** (elapsed time starting at $0.0\text{s}$ at the beginning of the audio stream) rather than document uptime, using `DOMHighResTimeStamp` would be semantically incorrect.
+
+3. **Numerical Precision:**
+   * A standard 64-bit IEEE 754 floating-point number (`double`) in seconds provides sub-nanosecond resolution across hours of continuous audio streaming, ensuring sample-accurate precision at any audio sample rate (e.g. 16 kHz to 96 kHz).
+
+4. **W3C TAG Design Principles Alignment:**
+   * This design adheres to the [W3C TAG Design Principles on Times and Dates](https://w3ctag.github.io/design-principles/#times-and-dates), ensuring consistency across media stream APIs on the web platform.
+
+---
+
 ### Proposed Behavior & Example Usage
 
-The `audioStartTime` and `audioEndTime` properties represent the audio duration bounds (in milliseconds) corresponding to the transcribed segment. If the underlying recognition engine backend does not support segment timestamps, these attributes return `null`.
+The `audioStartTime` and `audioEndTime` properties represent the audio duration bounds (in seconds) corresponding to the transcribed segment.
 
-Developers can programmatically compute processing latency by comparing `audioEndTime` against the standard DOM event generation timestamp (`Event.timeStamp`):
+Developers can programmatically compute processing latency by converting the stream timestamp to the document timeline and comparing against `Event.timeStamp`:
 
 ```javascript
 const recognition = new SpeechRecognition();
 recognition.continuous = true;
 recognition.interimResults = true;
 
+let audioOriginMs = 0;
+
+// 1. Capture the audio stream's start timestamp on the document timeline
+recognition.onaudiostart = (event) => {
+  audioOriginMs = event.timeStamp;
+};
+
 recognition.onresult = (event) => {
   const result = event.results[event.resultIndex];
 
-  if (result.audioEndTime !== null && result.audioEndTime !== undefined) {
-    // Calculate on-device processing latency
-    const processingLatencyMs = event.timeStamp - result.audioEndTime;
+  // 2. Convert stream-relative seconds to document timeline milliseconds
+  const absoluteAudioEndMs = audioOriginMs + (result.audioEndTime * 1000);
+
+  // 3. Calculate on-device processing latency
+  const processingLatencyMs = event.timeStamp - absoluteAudioEndMs;
 
-    // Trigger seamless failover to cloud backend if latency breaches acceptable threshold
-    if (processingLatencyMs > 1500) {
-      console.warn(`ASR processing lag detected (${processingLatencyMs}ms). Transitioning to cloud provider.`);
-      switchToCloudBackend();
-    }
+  // 4. Trigger seamless failover to cloud backend if latency breaches acceptable threshold
+  if (processingLatencyMs > 1500) {
+    console.warn(`ASR processing lag detected (${processingLatencyMs.toFixed(0)}ms). Transitioning to cloud provider.`);
+    switchToCloudBackend();
   }
 };
 
 recognition.start();
 ```
 
-## Converting Stream Timestamps to Document Time Origin
+---
 
-`audioStartTime` and `audioEndTime` are defined as media-local offsets in milliseconds relative to the start of the audio stream ($t = 0.0\text{ms}$).
+### Converting Stream Timestamps to Document Time Origin
 
-For real-time applications such as **live translation**, **subtitling overlays**, and **audio-visual sync**, developers often need to map these stream offsets to the document's global timeline (`DOMHighResTimeStamp` / `performance.now()`).
+`audioStartTime` and `audioEndTime` are defined as media-local offsets in seconds relative to the start of the audio stream ($t = 0.0\text{s}$).
 
-### Pattern: Capturing the Audio Timeline Origin
+For real-time applications such as **live translation**, **subtitling overlays**, and **audio-visual sync**, developers often need to map these stream offsets to the document's global timeline (`performance.timeOrigin` / `performance.now()`).
+
+#### Pattern: Capturing the Audio Timeline Origin
 
 To convert stream-relative timestamps to document time coordinates:
-1. Record the baseline timestamp when the `audiostart` event fires (`event.timeStamp` is a `DOMHighResTimeStamp` relative to `timeOrigin`).
-2. Add the result's `audioStartTime` and `audioEndTime` offsets to that baseline.
+1. Record the baseline timestamp when the `audiostart` event fires (`event.timeStamp` is in milliseconds relative to `timeOrigin`).
+2. Add the result's `audioStartTime` and `audioEndTime` offsets (converted to milliseconds) to that baseline:
 
-$$\text{absoluteStartTime} = \text{audioOrigin} + \text{result.audioStartTime}$$
-$$\text{absoluteEndTime} = \text{audioOrigin} + \text{result.audioEndTime}$$
+$$\text{absoluteStartTimeMs} = \text{audioOriginMs} + (\text{result.audioStartTime} \times 1000)$$
+$$\text{absoluteEndTimeMs} = \text{audioOriginMs} + (\text{result.audioEndTime} \times 1000)$$
 
-### Measuring Live Translation Latency Example
+#### Measuring Live Translation Latency Example
 
 In live speech translation workflows, measuring both **Speech-to-Text (STT) latency** and **Machine Translation (MT) end-to-end latency** is essential:
 
@@ -78,23 +111,22 @@ const recognition = new SpeechRecognition();
 recognition.continuous = true;
 recognition.interimResults = true;
 
-let audioOriginTime = 0;
+let audioOriginTimeMs = 0;
 
 // 1. Capture the audio stream's time origin on the document timeline
 recognition.onaudiostart = (event) => {
-  audioOriginTime = event.timeStamp;
+  audioOriginTimeMs = event.timeStamp;
 };
 
 recognition.onresult = async (event) => {
   const result = event.results[event.resultIndex];
-  if (result.audioEndTime === null || result.audioEndTime === undefined) return;
 
-  // 2. Convert stream offsets to document time origin coordinates
-  const absoluteAudioStart = audioOriginTime + result.audioStartTime;
-  const absoluteAudioEnd = audioOriginTime + result.audioEndTime;
+  // 2. Convert stream offsets (seconds) to document timeline (milliseconds)
+  const absoluteAudioStartMs = audioOriginTimeMs + (result.audioStartTime * 1000);
+  const absoluteAudioEndMs = audioOriginTimeMs + (result.audioEndTime * 1000);
 
   // 3. Compute ASR recognition latency
-  const asrLatencyMs = event.timeStamp - absoluteAudioEnd;
+  const asrLatencyMs = event.timeStamp - absoluteAudioEndMs;
 
   // 4. Perform live translation
   const text = result[0].transcript;
@@ -103,23 +135,44 @@ recognition.onresult = async (event) => {
   const translationEndTime = performance.now();
 
   // 5. Total end-to-end latency from speaker utterance to translated subtitle
-  const totalE2ELatencyMs = translationEndTime - absoluteAudioEnd;
+  const totalE2ELatencyMs = translationEndTime - absoluteAudioEndMs;
 
   console.log(`ASR Processing Time: ${asrLatencyMs.toFixed(1)}ms`);
   console.log(`Total Live Translation Delay: ${totalE2ELatencyMs.toFixed(1)}ms`);
 };
+
+recognition.start();
 ```
+
 ---
+
 ### Security and Privacy Considerations
 
 #### Fingerprinting Risk
-Exposing sub-millisecond or precise micro-architectural timing information enables hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a tracking vector for cross-origin user fingerprinting.
+Exposing sub-millisecond or precise micro-architectural timing information enables hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a potential tracking vector for cross-origin user fingerprinting.
 
 #### Mitigation Strategy
-To mitigate fingerprinting vectors, browser implementations MUST apply timestamp fuzzing and precision reduction before exposing timing attributes to web scripts. We propose mirroring the precision capping strategy used in [`HTMLMediaElement.currentTime`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/currentTime), rounding exposed timestamps to **2ms** precision (or matching the site-wide timer resolution policy).
+To mitigate potential side-channel and fingerprinting vectors:
+* **Precedent:** Implementations (such as Chromium) follow the security posture established by [`HTMLMediaElement.currentTime`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/currentTime), coarsening raw engine timestamps to discrete resolution buckets (e.g. 2ms resolution) in non-isolated execution contexts.
+* **Specification Precedent:** In accordance with [W3C High Resolution Time Level 3](https://www.w3.org/TR/hr-time-3/#privacy-security), the specification does not mandate a hardcoded quantization value, allowing user agents to adjust timer resolution or introduce jitter according to their security and cross-origin isolation policies.
+
+---
 
 ### Alternatives Considered
 
 - **Browser-Generated Warning Events (`onprocessinglag`):** Simple for web applications to catch, but fails to accommodate varying latency thresholds across different use cases (e.g. real-time meeting captioning requires <200ms latency, while dictation tools tolerate multi-second delays).
 - **Internal Processing Queue Metric (`queueDepth`):** Directly exposes engine backlogs, but is difficult to standardize across fragmented engine architectures, model types, and buffering strategies.
 - **Binary Status Flag (`isRealTime`):** Simple boolean check, but lacks numerical precision for applications seeking to track progressive latency degradation trendlines.
+- **Existing API Surfaces (Events):** Existing events were deemed insufficient because:
+  1. **`speechstart` and `speechend` Events:**
+     The Web Speech API specification defines `speechstart` and `speechend` events on the `SpeechRecognition` interface. However, these events cannot solve the continuous latency tracking problem:
+     * **Session-level vs. Result-level Granularity:** In continuous recognition mode, `speechstart` and `soundstart` fire once when voice activity is first detected at the beginning of the session. They do not fire for every individual phrase or sentence returned in subsequent `SpeechRecognitionResult` events.
+     * **Inequality with Result Audio Boundaries:** Because `speechstart` only marks initial voice activity, `speechstart.timeStamp` is not equal to `result.audioStartTime` for any subsequent utterance emitted throughout a session.
+     * **Fragile Event Correlation:** Even if engines fired `speechstart`/`speechend` around each phrase, associating separate asynchronous DOM events with streaming interim and final `SpeechRecognitionResult` objects requires complex, error-prone client-side state tracking (poor ergonomics).
+  2. **Overloading `event.timeStamp` on Result Events:**
+     Another alternative considered was modifying `event.timeStamp` on the `result` event to match the speech timing:
+     * **Eliminates Latency Calculation:** `event.timeStamp` indicates when the browser dispatched the DOM event on the document timeline. Keeping `event.timeStamp` intact while providing `result.audioEndTime` allows web applications to measure processing delay:
+       $$\text{latencyMs} = \text{event.timeStamp} - (\text{audioOriginMs} + \text{result.audioEndTime} \times 1000)$$
+     * Overwriting `event.timeStamp` would conflate acoustic timing with main-thread dispatch time, eliminating the ability to detect processing lag.
+
+Attaching `audioStartTime` and `audioEndTime` directly to `SpeechRecognitionResult` provides a 1:1 association between the recognized transcript text and its corresponding acoustic timeline.

From 68020a839f727c4e864327c8cce1a64dfab5de6f Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 17:47:12 -0700
Subject: [PATCH 3/9] Update speech-recognition-result-timestamps.md

---
 .../speech-recognition-result-timestamps.md   | 204 +++++++++---------
 1 file changed, 104 insertions(+), 100 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index ff85d63..645dbfa 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -4,107 +4,98 @@
 
 ### Problem
 
-The Web Speech API currently does not expose the start and end timestamps of the source audio corresponding to a given transcription result (`SpeechRecognitionResult`). This limitation creates two major challenges for API clients and end users:
+The Web Speech API currently does not expose the start and end timestamps of the source audio corresponding to a given transcription result (`SpeechRecognitionResult`). This limitation creates a major challenge for API clients requiring timeline association use cases:
 
-- **Timeline Association:** Developers cannot readily associate transcribed text with specific segments of the audio source, making it difficult to map generated captions to media timelines, audio tracks, or video frames.
-- **Latency Tracking & Backend Failover:** With the adoption of on-device Automatic Speech Recognition (ASR) to improve privacy and reduce server costs, processing performance becomes heavily dependent on local client hardware resources. The Web Speech API acts as a "black box" regarding local processing delays. Developers cannot programmatically calculate transcription latency or detect when on-device models fall behind real-time. This leads to poor user experiences (e.g. caption lag during live video conferencing) and deprives applications of the signal needed to seamlessly fail over to high-performance cloud backends.
+- **Timeline Association:** Developers cannot readily associate transcribed text with specific segments of the audio source, making it difficult to map generated captions to media timelines, audio tracks, or video frames:
+  - **Subtitling & Closed Captions:** Web applications cannot automatically generate synchronized subtitle tracks (e.g. WebVTT / SRT cues) because they lack the exact `[startTime, endTime]` boundaries for each phrase.
+  - **Interactive Meeting Transcripts ("Click-to-Seek"):** In recorded meetings, video lectures, and podcast players, applications cannot offer "click-to-seek" navigation—where clicking on a sentence or word in the transcript jumps media playback to that exact moment.
+  - **Live WebRTC Video Sync & Lip-Sync:** In real-time video conferencing (e.g. Google Meet), web applications cannot reliably synchronize live captions or translated subtitles with incoming video frames. Because DOM events conflate speech timing with processing and main-thread queuing delays, subtitles either lag behind speaker lip movement or disappear prematurely.
+  - **Text-Based Media Editing:** Web-based podcast and video editors cannot allow users to cut, splice, or re-time media segments by editing transcript text without knowing the corresponding audio boundaries.
+
+---
 
 ### Proposed Solution
 
-We propose extending the `SpeechRecognitionResult` interface to include `audioStartTime` and `audioEndTime` attributes.
+We propose extending the `SpeechRecognitionResult` interface with `speechStartTime` and `speechEndTime` attributes.
 
 #### Web IDL Definition
 
 ```webidl
 partial interface SpeechRecognitionResult {
-    // Start timestamp of the audio segment in seconds relative to the start of the audio stream (0.0s).
-    readonly attribute double audioStartTime;
+    // Start timestamp of the speech segment in seconds relative to the start of the audio stream (0.0s).
+    readonly attribute double speechStartTime;
 
-    // End timestamp of the audio segment in seconds relative to the start of the audio stream.
-    readonly attribute double audioEndTime;
+    // End timestamp of the speech segment in seconds relative to the start of the audio stream.
+    readonly attribute double speechEndTime;
 };
 ```
 
-### Choice of Time Representation: Seconds as `double`
+### Choice of Naming & Representation
 
-The timestamps `audioStartTime` and `audioEndTime` are defined as `double` representing **seconds**, rather than `DOMHighResTimeStamp` (milliseconds). This design choice is based on the following considerations:
+1. **Mirroring `speechstart` and `speechend`:**
+   * The names `speechStartTime` and `speechEndTime` mirror the existing `speechstart` and `speechend` events in the Web Speech API, clearly communicating that these timestamps bound the acoustic speech segment corresponding to the transcript hypothesis.
 
-1. **Consistency with Adjacent Web Audio & Media APIs:**
-   * In adjacent W3C media specifications, media-local stream timelines are universally represented in **seconds** as a `double`:
+2. **Seconds as `double` (Consistency with Adjacent Media APIs):**
+   * In adjacent W3C media specifications, media-local timelines are universally represented in **seconds** as a `double`:
      * **Web Audio API:** [`BaseAudioContext.currentTime`](https://webaudio.github.io/web-audio-api/#dom-baseaudiocontext-currenttime) (seconds)
      * **HTML Media Elements:** [`HTMLMediaElement.currentTime`](https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime) (seconds)
      * **AudioParam Scheduling:** [`AudioParam.setValueAtTime()`](https://webaudio.github.io/web-audio-api/#dom-audioparam-setvalueattime) (seconds)
-   * Using seconds ensures seamless interoperability when developers route audio between `<audio>`/`<video>` elements, Web Audio graphs, and `SpeechRecognition`, avoiding repetitive and error-prone unit conversions ($1000\times / \div 1000$).
-
-2. **Semantic Inaccuracy of `DOMHighResTimeStamp` for Media Streams:**
-   * Under the [W3C High Resolution Time Level 3](https://www.w3.org/TR/hr-time-3/#sec-domhighrestimestamp) specification, `DOMHighResTimeStamp` is strictly defined as a time coordinate in **milliseconds** measured relative to the global execution context's time origin (`performance.timeOrigin`).
-   * Because `audioStartTime` and `audioEndTime` represent a **media-local timeline** (elapsed time starting at $0.0\text{s}$ at the beginning of the audio stream) rather than document uptime, using `DOMHighResTimeStamp` would be semantically incorrect.
-
-3. **Numerical Precision:**
-   * A standard 64-bit IEEE 754 floating-point number (`double`) in seconds provides sub-nanosecond resolution across hours of continuous audio streaming, ensuring sample-accurate precision at any audio sample rate (e.g. 16 kHz to 96 kHz).
+   * Using seconds ensures seamless interoperability when routing audio between media elements, Web Audio graphs, and `SpeechRecognition`, avoiding repetitive unit conversions ($1000\times / \div 1000$).
 
-4. **W3C TAG Design Principles Alignment:**
-   * This design adheres to the [W3C TAG Design Principles on Times and Dates](https://w3ctag.github.io/design-principles/#times-and-dates), ensuring consistency across media stream APIs on the web platform.
+3. **Semantic Inaccuracy of `DOMHighResTimeStamp`:**
+   * Under [W3C High Resolution Time Level 3](https://www.w3.org/TR/hr-time-3/#sec-domhighrestimestamp), `DOMHighResTimeStamp` represents milliseconds relative to document uptime (`performance.timeOrigin`). Because `speechStartTime` and `speechEndTime` represent media-local stream offsets ($t = 0.0\text{s}$ at stream start), using `DOMHighResTimeStamp` would be semantically inaccurate.
 
 ---
 
-### Proposed Behavior & Example Usage
+### Proposed Behavior & Examples
 
-The `audioStartTime` and `audioEndTime` properties represent the audio duration bounds (in seconds) corresponding to the transcribed segment.
+The `speechStartTime` and `speechEndTime` properties represent the exact acoustic bounds (in seconds) of the recognized speech, measured from the start of the audio stream consumed by the recognizer ($t = 0.0\text{s}$, marked by `audiostart`).
 
-Developers can programmatically compute processing latency by converting the stream timestamp to the document timeline and comparing against `Event.timeStamp`:
+#### Example 1: Automated Subtitling & "Click-to-Seek" Navigation
+
+In this example, speech recognition results from an audio/video element are converted into WebVTT subtitle cues and interactive "click-to-seek" transcript links:
 
 ```javascript
+const mediaElement = document.querySelector('video');
+const track = mediaElement.addTextTrack('captions', 'English', 'en');
+track.mode = 'showing';
+
 const recognition = new SpeechRecognition();
 recognition.continuous = true;
-recognition.interimResults = true;
+recognition.interimResults = false;
 
-let audioOriginMs = 0;
-
-// 1. Capture the audio stream's start timestamp on the document timeline
-recognition.onaudiostart = (event) => {
-  audioOriginMs = event.timeStamp;
-};
+// Transcribe audio track from HTMLMediaElement
+const audioStream = mediaElement.captureStream();
+recognition.start(audioStream.getAudioTracks()[0]);
 
 recognition.onresult = (event) => {
-  const result = event.results[event.resultIndex];
-
-  // 2. Convert stream-relative seconds to document timeline milliseconds
-  const absoluteAudioEndMs = audioOriginMs + (result.audioEndTime * 1000);
-
-  // 3. Calculate on-device processing latency
-  const processingLatencyMs = event.timeStamp - absoluteAudioEndMs;
-
-  // 4. Trigger seamless failover to cloud backend if latency breaches acceptable threshold
-  if (processingLatencyMs > 1500) {
-    console.warn(`ASR processing lag detected (${processingLatencyMs.toFixed(0)}ms). Transitioning to cloud provider.`);
-    switchToCloudBackend();
+  for (let i = event.resultIndex; i < event.results.length; ++i) {
+    const result = event.results[i];
+    if (!result.isFinal) continue;
+
+    const transcriptText = result[0].transcript;
+
+    // 1. Create a synchronized WebVTT caption cue
+    const cue = new VTTCue(result.speechStartTime, result.speechEndTime, transcriptText);
+    track.addCue(cue);
+
+    // 2. Build interactive transcript element ("Click-to-Seek")
+    const transcriptItem = document.createElement('p');
+    transcriptItem.textContent = `[${result.speechStartTime.toFixed(1)}s] ${transcriptText}`;
+    transcriptItem.onclick = () => {
+      mediaElement.currentTime = result.speechStartTime;
+      mediaElement.play();
+    };
+    document.getElementById('transcript-container').appendChild(transcriptItem);
   }
 };
-
-recognition.start();
 ```
 
 ---
 
-### Converting Stream Timestamps to Document Time Origin
-
-`audioStartTime` and `audioEndTime` are defined as media-local offsets in seconds relative to the start of the audio stream ($t = 0.0\text{s}$).
-
-For real-time applications such as **live translation**, **subtitling overlays**, and **audio-visual sync**, developers often need to map these stream offsets to the document's global timeline (`performance.timeOrigin` / `performance.now()`).
-
-#### Pattern: Capturing the Audio Timeline Origin
-
-To convert stream-relative timestamps to document time coordinates:
-1. Record the baseline timestamp when the `audiostart` event fires (`event.timeStamp` is in milliseconds relative to `timeOrigin`).
-2. Add the result's `audioStartTime` and `audioEndTime` offsets (converted to milliseconds) to that baseline:
-
-$$\text{absoluteStartTimeMs} = \text{audioOriginMs} + (\text{result.audioStartTime} \times 1000)$$
-$$\text{absoluteEndTimeMs} = \text{audioOriginMs} + (\text{result.audioEndTime} \times 1000)$$
-
-#### Measuring Live Translation Latency Example
+#### Example 2: Synchronizing Captions with Live WebRTC Video Frames
 
-In live speech translation workflows, measuring both **Speech-to-Text (STT) latency** and **Machine Translation (MT) end-to-end latency** is essential:
+In live video conferencing, video frames delivered via `HTMLVideoElement.requestVideoFrameCallback()` contain timestamps relative to the document timeline. Developers can map `speechStartTime` and `speechEndTime` to the document timeline to render captions on the exact video frames when the speaker was talking:
 
 ```javascript
 const recognition = new SpeechRecognition();
@@ -113,35 +104,41 @@ recognition.interimResults = true;
 
 let audioOriginTimeMs = 0;
 
-// 1. Capture the audio stream's time origin on the document timeline
+// Capture the audio stream's start timestamp on the document timeline
 recognition.onaudiostart = (event) => {
   audioOriginTimeMs = event.timeStamp;
 };
 
-recognition.onresult = async (event) => {
-  const result = event.results[event.resultIndex];
-
-  // 2. Convert stream offsets (seconds) to document timeline (milliseconds)
-  const absoluteAudioStartMs = audioOriginTimeMs + (result.audioStartTime * 1000);
-  const absoluteAudioEndMs = audioOriginTimeMs + (result.audioEndTime * 1000);
+const activeCues = [];
 
-  // 3. Compute ASR recognition latency
-  const asrLatencyMs = event.timeStamp - absoluteAudioEndMs;
+recognition.onresult = (event) => {
+  const result = event.results[event.resultIndex];
 
-  // 4. Perform live translation
-  const text = result[0].transcript;
-  const translationStartTime = performance.now();
-  const translatedText = await translateService.translate(text, 'es');
-  const translationEndTime = performance.now();
+  // Convert stream offsets (seconds) to document timeline (milliseconds)
+  const startDocTimeMs = audioOriginTimeMs + (result.speechStartTime * 1000);
+  const endDocTimeMs   = audioOriginTimeMs + (result.speechEndTime * 1000);
+
+  // Update or add subtitle cue
+  activeCues[event.resultIndex] = {
+    text: result[0].transcript,
+    start: startDocTimeMs,
+    end: endDocTimeMs,
+    isFinal: result.isFinal
+  };
+};
 
-  // 5. Total end-to-end latency from speaker utterance to translated subtitle
-  const totalE2ELatencyMs = translationEndTime - absoluteAudioEndMs;
+// Frame-accurate rendering loop synchronized with WebRTC video playback
+function renderVideoSubtitleFrame(now, metadata) {
+  const currentVideoDocTimeMs = metadata.expectedDisplayTime; // document timeline
 
-  console.log(`ASR Processing Time: ${asrLatencyMs.toFixed(1)}ms`);
-  console.log(`Total Live Translation Delay: ${totalE2ELatencyMs.toFixed(1)}ms`);
-};
+  const currentCue = activeCues.find(
+    cue => currentVideoDocTimeMs >= cue.start && currentVideoDocTimeMs <= cue.end
+  );
 
-recognition.start();
+  subtitleOverlay.textContent = currentCue ? currentCue.text : '';
+  videoElement.requestVideoFrameCallback(renderVideoSubtitleFrame);
+}
+videoElement.requestVideoFrameCallback(renderVideoSubtitleFrame);
 ```
 
 ---
@@ -149,7 +146,7 @@ recognition.start();
 ### Security and Privacy Considerations
 
 #### Fingerprinting Risk
-Exposing sub-millisecond or precise micro-architectural timing information enables hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a potential tracking vector for cross-origin user fingerprinting.
+Exposing sub-millisecond acoustic timing can enable hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a potential tracking vector for cross-origin user fingerprinting.
 
 #### Mitigation Strategy
 To mitigate potential side-channel and fingerprinting vectors:
@@ -160,19 +157,26 @@ To mitigate potential side-channel and fingerprinting vectors:
 
 ### Alternatives Considered
 
-- **Browser-Generated Warning Events (`onprocessinglag`):** Simple for web applications to catch, but fails to accommodate varying latency thresholds across different use cases (e.g. real-time meeting captioning requires <200ms latency, while dictation tools tolerate multi-second delays).
-- **Internal Processing Queue Metric (`queueDepth`):** Directly exposes engine backlogs, but is difficult to standardize across fragmented engine architectures, model types, and buffering strategies.
-- **Binary Status Flag (`isRealTime`):** Simple boolean check, but lacks numerical precision for applications seeking to track progressive latency degradation trendlines.
-- **Existing API Surfaces (Events):** Existing events were deemed insufficient because:
-  1. **`speechstart` and `speechend` Events:**
-     The Web Speech API specification defines `speechstart` and `speechend` events on the `SpeechRecognition` interface. However, these events cannot solve the continuous latency tracking problem:
-     * **Session-level vs. Result-level Granularity:** In continuous recognition mode, `speechstart` and `soundstart` fire once when voice activity is first detected at the beginning of the session. They do not fire for every individual phrase or sentence returned in subsequent `SpeechRecognitionResult` events.
-     * **Inequality with Result Audio Boundaries:** Because `speechstart` only marks initial voice activity, `speechstart.timeStamp` is not equal to `result.audioStartTime` for any subsequent utterance emitted throughout a session.
-     * **Fragile Event Correlation:** Even if engines fired `speechstart`/`speechend` around each phrase, associating separate asynchronous DOM events with streaming interim and final `SpeechRecognitionResult` objects requires complex, error-prone client-side state tracking (poor ergonomics).
-  2. **Overloading `event.timeStamp` on Result Events:**
-     Another alternative considered was modifying `event.timeStamp` on the `result` event to match the speech timing:
-     * **Eliminates Latency Calculation:** `event.timeStamp` indicates when the browser dispatched the DOM event on the document timeline. Keeping `event.timeStamp` intact while providing `result.audioEndTime` allows web applications to measure processing delay:
-       $$\text{latencyMs} = \text{event.timeStamp} - (\text{audioOriginMs} + \text{result.audioEndTime} \times 1000)$$
-     * Overwriting `event.timeStamp` would conflate acoustic timing with main-thread dispatch time, eliminating the ability to detect processing lag.
-
-Attaching `audioStartTime` and `audioEndTime` directly to `SpeechRecognitionResult` provides a 1:1 association between the recognized transcript text and its corresponding acoustic timeline.
+- **Splitting `result` into `resultstart` and `resultend` Events:**
+  Splitting the single `result` event into separate `resultstart` and `resultend` DOM events (each carrying individual `event.timeStamp` values) was considered. However, localizing timestamps directly within `SpeechRecognitionResult` and `SpeechRecognitionResultList` is architecturally superior:
+  1. **Fragile Client State Management:** Requiring developers to correlate separate, asynchronous start and end events across streaming interim updates introduces a complex, race-prone state machine into web applications.
+  2. **Decoupling from Cumulative Results (`SpeechRecognitionResultList`):** In continuous recognition mode (`continuous = true`), the API maintains a cumulative list of all results generated during the session. If timing information is only emitted on transient events, historical results stored in `event.results` would have no localized timestamps. Developers would be forced to manually maintain a secondary timeline mapping in application code.
+  3. **Unnecessary Event Overhead for Limited Utility:** Adding yet another pair of events that client applications must process adds event-loop overhead and API surface complexity with limited practical use cases compared to self-contained result properties.
+  4. **Self-Contained Data Model:** Attaching `speechStartTime` and `speechEndTime` directly to `SpeechRecognitionResult` ensures that each transcript hypothesis is self-contained and atomically bound to its exact temporal audio segment, allowing results to be passed, cached, and manipulated independently of DOM event lifecycles.
+
+- **Using Existing `speechend` Event Timestamps:**
+  Relying on the `speechend` event timestamp was considered. While `speechend.timeStamp` (or the proposed start and end timestamps on `SpeechRecognitionResult`) could potentially be used for basic processing latency tracking, `speechend` is fundamentally insufficient for timeline association:
+  1. **No Start Timestamp:** `speechend` only marks when speech ended. It provides no start timestamp, making it impossible to determine when speech began, calculate utterance duration, or construct `[startTime, endTime]` subtitle cues.
+  2. **Session-Level vs. Per-Phrase Granularity:** In continuous recognition mode (`continuous = true`), `speechend` fires only once when the overall speech input terminates (e.g. at the end of a multi-minute session), not for individual sentences.
+  3. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
+  4. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.
+
+- **Overloading `SpeechRecognitionEvent.timeStamp`:**
+  Modifying `event.timeStamp` on the `result` event was considered, but fails because:
+  1. **Container of Multiple Results:** In continuous recognition, `event.results` is a cumulative list of multiple `SpeechRecognitionResult` objects. A single timestamp on the outer event wrapper cannot describe the independent start and end boundaries of each historical phrase in the list.
+  2. **Breaks DOM Standards:** `Event.timeStamp` is specified across the web platform as the time the DOM event was dispatched. Overwriting it disrupts developer expectations and deviates from standard DOM event semantics.
+
+- **Browser-Generated Warning Events (`onprocessinglag`):**
+  Simple to catch, but fails to solve timeline association and cannot accommodate varying application needs.
+
+Attaching `speechStartTime` and `speechEndTime` directly to `SpeechRecognitionResult` provides an atomic, 1:1 binding between the recognized transcript text and its corresponding acoustic timeline.

From 7525e4153ba3cb98b1a47f8371c717dfd0373131 Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 18:11:45 -0700
Subject: [PATCH 4/9] Revise speech recognition result timestamp explanations

Removed sections discussing the limitations of `speechend` and the overloading of `SpeechRecognitionEvent.timeStamp`. Added a note about attaching `speechStartTime` and `speechEndTime` to `SpeechRecognitionResult` for better timeline association.
---
 explainers/speech-recognition-result-timestamps.md | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index 645dbfa..8cac45b 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -171,12 +171,4 @@ To mitigate potential side-channel and fingerprinting vectors:
   3. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
   4. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.
 
-- **Overloading `SpeechRecognitionEvent.timeStamp`:**
-  Modifying `event.timeStamp` on the `result` event was considered, but fails because:
-  1. **Container of Multiple Results:** In continuous recognition, `event.results` is a cumulative list of multiple `SpeechRecognitionResult` objects. A single timestamp on the outer event wrapper cannot describe the independent start and end boundaries of each historical phrase in the list.
-  2. **Breaks DOM Standards:** `Event.timeStamp` is specified across the web platform as the time the DOM event was dispatched. Overwriting it disrupts developer expectations and deviates from standard DOM event semantics.
-
-- **Browser-Generated Warning Events (`onprocessinglag`):**
-  Simple to catch, but fails to solve timeline association and cannot accommodate varying application needs.
-
 Attaching `speechStartTime` and `speechEndTime` directly to `SpeechRecognitionResult` provides an atomic, 1:1 binding between the recognized transcript text and its corresponding acoustic timeline.

From 1c867319e127bd7b0acca691a4aa308c12d715a5 Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 18:15:42 -0700
Subject: [PATCH 5/9] Revise speech recognition result timestamp considerations

Updated the discussion on using existing event timestamps for speech recognition results, highlighting their limitations and the benefits of attaching timestamps directly to SpeechRecognitionResult.
---
 .../speech-recognition-result-timestamps.md       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index 8cac45b..d9ed9b7 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -164,11 +164,10 @@ To mitigate potential side-channel and fingerprinting vectors:
   3. **Unnecessary Event Overhead for Limited Utility:** Adding yet another pair of events that client applications must process adds event-loop overhead and API surface complexity with limited practical use cases compared to self-contained result properties.
   4. **Self-Contained Data Model:** Attaching `speechStartTime` and `speechEndTime` directly to `SpeechRecognitionResult` ensures that each transcript hypothesis is self-contained and atomically bound to its exact temporal audio segment, allowing results to be passed, cached, and manipulated independently of DOM event lifecycles.
 
-- **Using Existing `speechend` Event Timestamps:**
-  Relying on the `speechend` event timestamp was considered. While `speechend.timeStamp` (or the proposed start and end timestamps on `SpeechRecognitionResult`) could potentially be used for basic processing latency tracking, `speechend` is fundamentally insufficient for timeline association:
-  1. **No Start Timestamp:** `speechend` only marks when speech ended. It provides no start timestamp, making it impossible to determine when speech began, calculate utterance duration, or construct `[startTime, endTime]` subtitle cues.
-  2. **Session-Level vs. Per-Phrase Granularity:** In continuous recognition mode (`continuous = true`), `speechend` fires only once when the overall speech input terminates (e.g. at the end of a multi-minute session), not for individual sentences.
-  3. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
-  4. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.
-
-Attaching `speechStartTime` and `speechEndTime` directly to `SpeechRecognitionResult` provides an atomic, 1:1 binding between the recognized transcript text and its corresponding acoustic timeline.
+- **Using Existing VAD (`speechstart` / `speechend`) and `result` Event Timestamps:**
+  Relying on existing Voice Activity Detection (VAD) events (`speechstart` / `speechend`) and `result` event timestamps was considered. While `speechend.timeStamp` (or the proposed start and end timestamps on `SpeechRecognitionResult`) could potentially be used for basic processing latency tracking, existing events are fundamentally insufficient for timeline association:
+  1. **Cannot Capture Start Times in Continuous Recognition:** Neither VAD events nor `result` events capture the acoustic start timestamp of interim or final results that occur in the middle of a continuous recognition session. In continuous recognition mode (`continuous = true`), `speechstart` fires only once at the beginning of the entire session, leaving all subsequent phrases without an acoustic start boundary.
+  2. **No Start Timestamp on `speechend` or `result` Events:** `speechend` only marks when speech activity ended, and `result.timeStamp` only indicates when the DOM event was dispatched. Neither provides the acoustic start time needed to compute utterance duration or construct `[startTime, endTime]` subtitle cues.
+  3. **Session-Level vs. Per-Phrase Granularity:** In continuous recognition mode (`continuous = true`), `speechend` fires only once when the overall speech input terminates (e.g. at the end of a multi-minute session), not for individual sentences.
+  4. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
+  5. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.

From a6558f6adf0c383be24d6745998ba7e6864c8edc Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 18:23:27 -0700
Subject: [PATCH 6/9] Refine explanation on VAD and result event timestamps

Removed redundant points about interim results and trailing silence skew from the explanation of existing VAD and result event timestamps.
---
 explainers/speech-recognition-result-timestamps.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index d9ed9b7..43b2d81 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -168,6 +168,5 @@ To mitigate potential side-channel and fingerprinting vectors:
   Relying on existing Voice Activity Detection (VAD) events (`speechstart` / `speechend`) and `result` event timestamps was considered. While `speechend.timeStamp` (or the proposed start and end timestamps on `SpeechRecognitionResult`) could potentially be used for basic processing latency tracking, existing events are fundamentally insufficient for timeline association:
   1. **Cannot Capture Start Times in Continuous Recognition:** Neither VAD events nor `result` events capture the acoustic start timestamp of interim or final results that occur in the middle of a continuous recognition session. In continuous recognition mode (`continuous = true`), `speechstart` fires only once at the beginning of the entire session, leaving all subsequent phrases without an acoustic start boundary.
   2. **No Start Timestamp on `speechend` or `result` Events:** `speechend` only marks when speech activity ended, and `result.timeStamp` only indicates when the DOM event was dispatched. Neither provides the acoustic start time needed to compute utterance duration or construct `[startTime, endTime]` subtitle cues.
-  3. **Session-Level vs. Per-Phrase Granularity:** In continuous recognition mode (`continuous = true`), `speechend` fires only once when the overall speech input terminates (e.g. at the end of a multi-minute session), not for individual sentences.
-  4. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
-  5. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.
+  3. **No Support for Interim Results:** While a speaker is actively talking mid-sentence, `speechend` cannot fire, leaving streaming captions without timing information.
+  4. **Endpointer Trailing Silence Skew:** Voice Activity Detection (VAD) endpointers only fire `speechend` after observing 500ms–1500ms of trailing silence. This introduces non-speech padding into the timestamp, degrading audio alignment.

From 872c089c9a435804b8f9ee860ade2a8aef40923f Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 20:29:10 -0700
Subject: [PATCH 7/9] Clarify timestamp limitations and synchronization example

Updated the explanation of timestamp limitations in the Web Speech API and provided a clearer example of synchronizing captions with live video frames, including a playout buffer delay for ASR processing.
---
 .../speech-recognition-result-timestamps.md       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index 43b2d81..fa0a9c4 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -5,8 +5,6 @@
 ### Problem
 
 The Web Speech API currently does not expose the start and end timestamps of the source audio corresponding to a given transcription result (`SpeechRecognitionResult`). This limitation creates a major challenge for API clients requiring timeline association use cases:
-
-- **Timeline Association:** Developers cannot readily associate transcribed text with specific segments of the audio source, making it difficult to map generated captions to media timelines, audio tracks, or video frames:
   - **Subtitling & Closed Captions:** Web applications cannot automatically generate synchronized subtitle tracks (e.g. WebVTT / SRT cues) because they lack the exact `[startTime, endTime]` boundaries for each phrase.
   - **Interactive Meeting Transcripts ("Click-to-Seek"):** In recorded meetings, video lectures, and podcast players, applications cannot offer "click-to-seek" navigation—where clicking on a sentence or word in the transcript jumps media playback to that exact moment.
   - **Live WebRTC Video Sync & Lip-Sync:** In real-time video conferencing (e.g. Google Meet), web applications cannot reliably synchronize live captions or translated subtitles with incoming video frames. Because DOM events conflate speech timing with processing and main-thread queuing delays, subtitles either lag behind speaker lip movement or disappear prematurely.
@@ -95,7 +93,7 @@ recognition.onresult = (event) => {
 
 #### Example 2: Synchronizing Captions with Live WebRTC Video Frames
 
-In live video conferencing, video frames delivered via `HTMLVideoElement.requestVideoFrameCallback()` contain timestamps relative to the document timeline. Developers can map `speechStartTime` and `speechEndTime` to the document timeline to render captions on the exact video frames when the speaker was talking:
+In live video conferencing, video frames delivered via `HTMLVideoElement.requestVideoFrameCallback()` contain presentation timestamps (`metadata.expectedDisplayTime`) relative to the document timeline. When video playout is buffered to accommodate ASR processing, developers can map `speechStartTime` and `speechEndTime` to the document timeline to render captions on the exact video frames when the speaker was talking:
 
 ```javascript
 const recognition = new SpeechRecognition();
@@ -109,16 +107,17 @@ recognition.onaudiostart = (event) => {
   audioOriginTimeMs = event.timeStamp;
 };
 
+// Playout buffer delay (e.g. 1.0s) to allow ASR processing time before video frames are presented
+const PLAYOUT_DELAY_MS = 1000;
 const activeCues = [];
 
 recognition.onresult = (event) => {
   const result = event.results[event.resultIndex];
 
-  // Convert stream offsets (seconds) to document timeline (milliseconds)
-  const startDocTimeMs = audioOriginTimeMs + (result.speechStartTime * 1000);
-  const endDocTimeMs   = audioOriginTimeMs + (result.speechEndTime * 1000);
+  // Convert stream offsets to document timeline, offset by the playout buffer
+  const startDocTimeMs = audioOriginTimeMs + (result.speechStartTime * 1000) + PLAYOUT_DELAY_MS;
+  const endDocTimeMs   = audioOriginTimeMs + (result.speechEndTime * 1000) + PLAYOUT_DELAY_MS;
 
-  // Update or add subtitle cue
   activeCues[event.resultIndex] = {
     text: result[0].transcript,
     start: startDocTimeMs,
@@ -146,7 +145,7 @@ videoElement.requestVideoFrameCallback(renderVideoSubtitleFrame);
 ### Security and Privacy Considerations
 
 #### Fingerprinting Risk
-Exposing sub-millisecond acoustic timing can enable hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a potential tracking vector for cross-origin user fingerprinting.
+Exposing sub-millisecond or precise micro-architectural timing information enables hardware profiling (measuring CPU execution speed, thermal throttling, and system load), creating a potential tracking vector for cross-origin user fingerprinting.
 
 #### Mitigation Strategy
 To mitigate potential side-channel and fingerprinting vectors:

From 633565dfa04f3987d78b44a383787a02081e546e Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Mon, 31 Aug 2026 21:56:39 -0700
Subject: [PATCH 8/9] Enhance example with interactive demo link

Added a link to an interactive demo for synchronizing captions with live WebRTC video frames.
---
 explainers/speech-recognition-result-timestamps.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index fa0a9c4..3f7d4d6 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -95,6 +95,8 @@ recognition.onresult = (event) => {
 
 In live video conferencing, video frames delivered via `HTMLVideoElement.requestVideoFrameCallback()` contain presentation timestamps (`metadata.expectedDisplayTime`) relative to the document timeline. When video playout is buffered to accommodate ASR processing, developers can map `speechStartTime` and `speechEndTime` to the document timeline to render captions on the exact video frames when the speaker was talking:
 
+*(See the [interactive live video lip-sync demo](https://alan33d.github.io/web-speech-demos/webrtc_caption_sync_test.html) for a working prototype of this pattern).*
+
 ```javascript
 const recognition = new SpeechRecognition();
 recognition.continuous = true;

From 3c123936e18604544c1024d0e7873e8542f1e481 Mon Sep 17 00:00:00 2001
From: "Alan Ding (Google)" <alan33d@users.noreply.github.com>
Date: Tue, 1 Sep 2026 18:24:03 -0700
Subject: [PATCH 9/9] Revise example for live video conferencing captions

Updated example to clarify live video conferencing with accurate caption synchronization and added details on handling captions during live calls and post-meeting replay.
---
 .../speech-recognition-result-timestamps.md   | 68 ++++++++++---------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/explainers/speech-recognition-result-timestamps.md b/explainers/speech-recognition-result-timestamps.md
index 3f7d4d6..2aca9d1 100644
--- a/explainers/speech-recognition-result-timestamps.md
+++ b/explainers/speech-recognition-result-timestamps.md
@@ -91,55 +91,57 @@ recognition.onresult = (event) => {
 
 ---
 
-#### Example 2: Synchronizing Captions with Live WebRTC Video Frames
+#### Example 2: Live Video Conferencing (Live Display + Recording Timeline Association)
 
-In live video conferencing, video frames delivered via `HTMLVideoElement.requestVideoFrameCallback()` contain presentation timestamps (`metadata.expectedDisplayTime`) relative to the document timeline. When video playout is buffered to accommodate ASR processing, developers can map `speechStartTime` and `speechEndTime` to the document timeline to render captions on the exact video frames when the speaker was talking:
+In zero-delay live conferencing (such as Google Meet), captions are displayed immediately upon arrival during the live call to minimize perceived lag. Simultaneously, `speechStartTime` and `speechEndTime` are preserved in the meeting transcript so that the exported recording video track can be rendered with frame-accurate subtitle alignment:
 
-*(See the [interactive live video lip-sync demo](https://alan33d.github.io/web-speech-demos/webrtc_caption_sync_test.html) for a working prototype of this pattern).*
+*(See the [interactive live meeting and recording demo](https://alan33d.github.io/web-speech-demos/live_meeting_recording_demo.html) for a working implementation of this pattern).*
 
 ```javascript
 const recognition = new SpeechRecognition();
 recognition.continuous = true;
 recognition.interimResults = true;
 
-let audioOriginTimeMs = 0;
+// 1. Live Call: Display captions immediately as results arrive
+let hideTimeout = null;
+const liveSubtitleOverlay = document.getElementById('live-captions');
 
-// Capture the audio stream's start timestamp on the document timeline
-recognition.onaudiostart = (event) => {
-  audioOriginTimeMs = event.timeStamp;
-};
-
-// Playout buffer delay (e.g. 1.0s) to allow ASR processing time before video frames are presented
-const PLAYOUT_DELAY_MS = 1000;
-const activeCues = [];
+// 2. Meeting Transcript: Archive finalized results with exact audio boundaries
+const meetingTranscript = [];
 
 recognition.onresult = (event) => {
   const result = event.results[event.resultIndex];
-
-  // Convert stream offsets to document timeline, offset by the playout buffer
-  const startDocTimeMs = audioOriginTimeMs + (result.speechStartTime * 1000) + PLAYOUT_DELAY_MS;
-  const endDocTimeMs   = audioOriginTimeMs + (result.speechEndTime * 1000) + PLAYOUT_DELAY_MS;
-
-  activeCues[event.resultIndex] = {
-    text: result[0].transcript,
-    start: startDocTimeMs,
-    end: endDocTimeMs,
-    isFinal: result.isFinal
-  };
+  const transcriptText = result[0].transcript;
+
+  // Render live captions immediately on screen
+  liveSubtitleOverlay.textContent = transcriptText;
+  liveSubtitleOverlay.classList.toggle('interim', !result.isFinal);
+
+  // Keep final captions on screen for a 3-second reading persistence window
+  clearTimeout(hideTimeout);
+  if (result.isFinal) {
+    hideTimeout = setTimeout(() => {
+      liveSubtitleOverlay.textContent = '';
+    }, 3000);
+
+    // Archive the finalized sentence with exact acoustic timestamps
+    meetingTranscript.push({
+      text: transcriptText,
+      startTime: result.speechStartTime,
+      endTime: result.speechEndTime
+    });
+  }
 };
 
-// Frame-accurate rendering loop synchronized with WebRTC video playback
-function renderVideoSubtitleFrame(now, metadata) {
-  const currentVideoDocTimeMs = metadata.expectedDisplayTime; // document timeline
-
-  const currentCue = activeCues.find(
-    cue => currentVideoDocTimeMs >= cue.start && currentVideoDocTimeMs <= cue.end
-  );
+// 3. Post-Meeting Replay: Synchronize caption cues with the recorded meeting video
+function initializeRecordingCaptions(recordedVideoElement) {
+  const captionTrack = recordedVideoElement.addTextTrack('captions', 'Meeting Captions');
+  captionTrack.mode = 'showing';
 
-  subtitleOverlay.textContent = currentCue ? currentCue.text : '';
-  videoElement.requestVideoFrameCallback(renderVideoSubtitleFrame);
+  for (const entry of meetingTranscript) {
+    captionTrack.addCue(new VTTCue(entry.startTime, entry.endTime, entry.text));
+  }
 }
-videoElement.requestVideoFrameCallback(renderVideoSubtitleFrame);
 ```
 
 ---