feat: LSDV-3028: Audio v3 dual channel render (#1185)

* Revert "Revert feat: DEV-4034: Rendering performance improvements for large-duration audio (#1138) (#1160)"

This reverts commit d84c38e.

* fix: LSDV-4532: Correct decoded audio data offset

* update the removal grace period for audio cached decoders

* only pause playback if it was playing in disconnect, remove redundant methods

* force buffer audio playback by playing then pausing, and ensure media has ability to playback through buffer

* adding comment about the duration offset start for decoding audio

* feat: DEV-3028: Audio v3 dual channel render

* fix: LSDV-4532: Audio sometimes goes missing during playback

* fix: LSDV-4532: Waveform sometimes blank when toggling visibility off then on

* fix: LSDV-4532: Sound sometimes delayed to playback

* when layer visibility changes we now fire renderAvailableChannels

* add crossorigin anonymous to audio element

* testing cors issue

* changing back to proper src

* fix multi channel split by using forked lib with a patch fix

* fix: LSDV-4532: Adding more tests fixing playbackRate setting

* default to splitchannel if possible

* update audio-file-decoder

* adding zoom test

* fix the number of chunks calculation to account for the larger sized samples returned from the decoder in splitchannel

* missing the calculation to augment chunk size when channel count is greater than one

* ensure a minwaveheight is adhered to, so that each channel wave is visible

* fix default waveHeight when non splitchannel

* fix initial render, and layerUpdated causing contentious renders that can sometimes cause unexpected results in layers

* fix initial render of single channel, original height waveform

* ensure the splitchannel worker is lazy to init, and can be destroyed when not in use

* fix partial waveform render optimization for large data point renders that occur across multiple channels

* change Audio tag param from minwaveheight to more aptly named waveheight

* removing console log

* adding ff to allow audio v3 multichannel support be default on

---------

Co-authored-by: Yousif Yassi <yousif@heartex.com>

Author: bmartel

e2e/fragments/AtAudioView.js

@@ -56,20 +56,71 @@ module.exports = {
     I.wait(1);
   },
 
-  clickAt(x) {
+  /**
+   * Click on the Audio waveform at the given x coordinate, and optional y coordinate.
+   * @example
+   * await AtAudioView.lookForStage();
+   * AtAudioView.clickAt(50);
+   * @param {number} x
+   * @param {number} [y=undefined] - if not provided, will click at the center of the waveform
+   */
+  clickAt(x, y) {
+    y = y !== undefined ? y : this._stageBbox.height / 2;
     I.scrollPageToTop();
-    I.clickAt(this._stageBbox.x + x, this._stageBbox.y + this._stageBbox.height / 2);
+    I.clickAt(this._stageBbox.x + x, this._stageBbox.y + y);
     I.wait(1); // We gotta  wait here because clicks on the canvas are not processed immediately
   },
 
+  /**
+   * Toggle the audio control menu. This houses the volume slider and mute button.
+   */
   toggleControlsMenu() {
     I.click(this._controlMenuSelector);
   },
 
+  /**
+   * Toggle the audio settings menu. This houses the playback speed slider, amplitude slider, and interface visibility toggles.
+   */
   toggleSettingsMenu() {
     I.click(this._settingsMenuSelector);
   },
 
+  /**
+   * Zooms in the Audio Waveform by using the mouse wheel at the given relative position.
+   * @param {number} deltaY
+   * @param {Object} [atPoint] - Point where the wheel action will be called 
+   * @param {number} [atPoint.x=0.5] - relative X coordinate
+   * @param {number} [atPoint.y=0.5] - relative Y coordinate
+   * @returns {Promise<void>}
+   *
+   * @example
+   * // zoom in
+   * await AtAudioView.zoomToPoint(-100, { x: .01 });
+   * // zoom out
+   * await AtAudioView.zoomToPoint(100);
+   */
+  async zoomToPoint(deltaY, atPoint = { x: 0.5, y: 0.5 }) {
+    const { x = 0.5, y = 0.5 } = atPoint;
+
+    I.scrollPageToTop();
+
+    const stageBBox = await I.grabElementBoundingRect(this._stageSelector);
+
+    I.clickAt(stageBBox.x + stageBBox.width * x, stageBBox.y + stageBBox.height * y); // click to focus the canvas
+
+    I.pressKeyDown('Control');
+    I.mouseWheel({ deltaY });
+    I.pressKeyUp('Control');
+  },
+
+  /**
+   * Asserts the current volume of the audio player the slider, input and the audio player.
+   * @param {number} value - volume in the range [0, 100]
+   * @returns {Promise<void>}
+   *
+   * @example
+   * await AtAudioView.seeVolume(50);
+   */
   async seeVolume(value) {
     this.toggleControlsMenu();
     I.seeInField(this._volumeInputSelector, value);
@@ -85,12 +136,27 @@ module.exports = {
     this.toggleControlsMenu();
   },
 
+  /**
+   * Sets the volume of the audio player via the input.
+   * @param {number} value - volume in the range [0, 100]
+   *
+   * @example
+   * AtAudioView.setVolumeInput(50);
+   */
   setVolumeInput(value) {
     this.toggleControlsMenu();
     I.fillField(this._volumeInputSelector, value);
     this.toggleControlsMenu();
   },
 
+  /**
+   * Asserts the current playback speed of the audio player the slider, input and the audio player.
+   * @param {number} value - speed in the range [0.5, 2.5]
+   * @returns {Promise<void>}
+   *
+   * @example
+   * await AtAudioView.seePlaybackSpeed(1.5);
+   */
   async seePlaybackSpeed(value) {
     this.toggleSettingsMenu();
 
@@ -103,12 +169,27 @@ module.exports = {
     this.toggleSettingsMenu();
   },
 
+  /**
+   * Sets the playback speed of the audio player via the input.
+   * @param {number} value - speed in the range [0.5, 2.5]
+   *
+   * @example
+   * AtAudioView.setPlaybackSpeedInput(1.5);
+   */
   setPlaybackSpeedInput(value) {
     this.toggleSettingsMenu();
     I.fillField(this._playbackSpeedInputSelector, value);
     this.toggleSettingsMenu();
   },
 
+  /**
+   * Asserts the current amplitude of the audio player the slider, and the input.
+   * @param {number} value - amplitude (y-axis zoom) in the range [1, 150]
+   * @returns {Promise<void>}
+   *
+   * @example
+   * await AtAudioView.seeAmplitude(10);
+   */
   async seeAmplitude(value) {
     this.toggleSettingsMenu();
 
@@ -118,6 +199,13 @@ module.exports = {
     this.toggleSettingsMenu();
   },
 
+  /**
+   * Sets the amplitude of the audio player via the input.
+   * @param {number} value - speed in the range [1, 150]
+   *
+   * @example
+   * AtAudioView.setAmplitudeInput(10);
+   */
   setAmplitudeInput(value) {
     this.toggleSettingsMenu();
     I.fillField(this._amplitudeInputSelector, value);
@@ -143,10 +231,14 @@ module.exports = {
 
     assert.equal(selectedChoice, 'Positive');
   },
-
+  
+  /**
+   * Asserts whether the audio player is reporting as paused.
+   * @returns {Promise<void>}
+   */
   async seeIsPlaying(playing) {
     const isPaused = await I.grabAttributeFrom(this._audioElementSelector, 'paused');
 
-    assert.equal(!isPaused, playing, 'Audio is not playing');
+    assert.equal(!isPaused, playing, playing ? 'Audio is not playing' : 'Audio is playing');
   },
 };

e2e/tests/audio/audio-controls.test.js

@@ -1,6 +1,8 @@
 
 /* global Feature, Scenario */
 
+const assert = require('assert');
+
 Feature('Audio Controls');
 
 const config = `
@@ -56,12 +58,23 @@ const annotations = [
 
 const params = { annotations: [{ id: 'test', result: annotations }], config, data };
 
-Scenario('Check the audio controls work', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
+Scenario('Check the audio controls work', async function({ I, LabelStudio, ErrorsCollector, AtAudioView, AtSidebar }) {
+  async function doNotSeeErrors() {
+    await I.wait(2);
+    const errors = await ErrorsCollector.grabErrors();
+
+    if (errors.length) {
+      assert.fail(`Got an error: ${errors[0]}`);
+    }
+  }
+
   LabelStudio.setFeatureFlags({
     ff_front_dev_2715_audio_3_280722_short: true,
   });
   I.amOnPage('/');
 
+  await ErrorsCollector.run();
+
   LabelStudio.init(params);
 
   await AtAudioView.waitForAudio();
@@ -115,4 +128,10 @@ Scenario('Check the audio controls work', async function({ I, LabelStudio, AtAud
   AtAudioView.clickPauseButton();
 
   await AtAudioView.seeIsPlaying(false);
+
+  I.say('Check the waveform can be zoomed without error');
+
+  await AtAudioView.zoomToPoint(-120);
+
+  await doNotSeeErrors();
 });

package.json

@@ -86,7 +86,7 @@
   ],
   "dependencies": {
     "@thi.ng/rle-pack": "^2.1.6",
-    "audio-file-decoder": "^2.3.0",
+    "@martel/audio-file-decoder": "2.3.15",
     "babel-preset-react-app": "^9.1.1",
     "d3": "^5.16.0",
     "magic-wand-js": "^1.0.0",

scripts/postinstall.sh

@@ -1 +1 @@
-cp ./node_modules/audio-file-decoder/decode-audio.wasm ./node_modules/audio-file-decoder/dist/decode-audio.wasm
+cp ./node_modules/@martel/audio-file-decoder/decode-audio.wasm ./node_modules/@martel/audio-file-decoder/dist/decode-audio.wasm

src/lib/AudioUltra/Common/Worker/index.ts

@@ -46,7 +46,8 @@ export class ComputeWorker {
       };
 
       self.addEventListener('message', (e) => {
-        if (e.data) return;
+        if (!e.data) return;
+
         const { data, type, eventId } = e.data;
 
         switch(type) {
@@ -69,7 +70,7 @@ export class ComputeWorker {
       type: 'compute',
     }, true);
 
-    return result?.data;
+    return result?.data?.result?.data;
   }
 
   async precompute(data: Record<string, any>) {
@@ -105,6 +106,7 @@ export class ComputeWorker {
       if (waitResponse) {
         const resolver = (e: MessageEvent) => {
           if (eventId === e.data.eventId) {
+            worker.removeEventListener('message', resolver);
             resolve(e);
           }
         };

src/lib/AudioUltra/Media/AudioDecoder.ts

@@ -1,9 +1,11 @@
-import { AudioDecoderWorker, getAudioDecoderWorker } from 'audio-file-decoder';
+import { AudioDecoderWorker, getAudioDecoderWorker } from '@martel/audio-file-decoder';
 // eslint-disable-next-line
 // @ts-ignore
-import DecodeAudioWasm from 'audio-file-decoder/decode-audio.wasm';
+import DecodeAudioWasm from '@martel/audio-file-decoder/decode-audio.wasm';
 import { Events } from '../Common/Events';
 import { clamp, info } from '../Common/Utils';
+import { SplitChannel } from './SplitChannel';
+
 
 const DURATION_CHUNK_SIZE = 60 * 30; // 30 minutes
 
@@ -14,7 +16,7 @@ interface AudioDecoderEvents {
 }
 
 export class AudioDecoder extends Events<AudioDecoderEvents> {
-  chunks?: Float32Array[];
+  chunks?: Float32Array[][];
   private cancelled = false;
   private decodeId = 0; // if id=0, decode is not in progress
   private worker: AudioDecoderWorker | undefined;
@@ -50,14 +52,14 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
 
   get dataLength() {
     if (this.chunks && !this._dataLength) {
-      this._dataLength = this.chunks?.reduce((a, b) => a + b.length, 0) ?? 0;
+      this._dataLength = (this.chunks?.reduce((a, b) => a + b.reduce((_a, _b) => _a + _b.length, 0), 0) ?? 0) / this._channelCount;
     }
     return this._dataLength;
   }
 
   get dataSize() {
     if (this.chunks && !this._dataSize) {
-      this._dataSize = this.chunks?.reduce((a, b) => a + b.byteLength, 0) ?? 0;
+      this._dataSize = (this.chunks?.reduce((a, b) => a + b.reduce((_a, _b) => _a + _b.byteLength, 0), 0) ?? 0) / this._channelCount;
     }
     return this._dataSize;
   }
@@ -111,10 +113,28 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
   }
 
   /**
-   * Total number of chunks to decode
+   * Total number of chunks to decode.
+   *
+   * This is used to allocate the number of chunks to decode and to calculate the progress.
+   * Influenced by the number of channels and the duration of the audio file, as this will cause errors
+   * if the decoder tries to decode and return too much data at once.
+   *
+   * @example
+   * 1hour 44.1kHz 2ch = 1 * 60 * 60 * 44100 * 2 = 158760000 samples -> 4 chunks (39690000 samples/chunk)
+   * 1hour 44.1kHz 1ch = 1 * 60 * 60 * 44100 * 1 = 79380000 samples -> 2 chunks (39690000 samples/chunk)
+   */
+  getTotalChunks() {
+    return Math.ceil((this._duration  * this._channelCount) / DURATION_CHUNK_SIZE);
+  }
+
+  /**
+   * Total size in duration seconds per chunk to decode.
+   *
+   * This is used to work out the number of samples to decode per chunk, as the decoder will
+   * return an error if too much data is requested at once. This is influenced by the number of channels.
    */
-  getTotalChunks(duration: number) {
-    return Math.ceil(duration / DURATION_CHUNK_SIZE);
+  getChunkDuration() {
+    return DURATION_CHUNK_SIZE / this._channelCount;
   }
 
   /**
@@ -154,17 +174,22 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
     // This is a shared promise which will be observed by all instances of the same source
     this.decodingPromise =  new Promise(resolve => (this.decodingResolve = resolve as any));
 
+    let splitChannels: SplitChannel | undefined = undefined;
+
     try {
       // Set the worker instance and resolve the decoder promise
-      this._channelCount = this.worker.channelCount;
+      this._channelCount = options?.multiChannel ? this.worker.channelCount : 1;
       this._sampleRate = this.worker.sampleRate;
       this._duration = this.worker.duration;
 
+
       let chunkIndex = 0;
-      const totalChunks = this.getTotalChunks(this.worker.duration);
+      const totalChunks = this.getTotalChunks();
       const chunkIterator = this.chunkDecoder(options);
 
-      const chunks = Array.from({ length: totalChunks }) as Float32Array[];
+      splitChannels = this._channelCount > 1 ? new SplitChannel(this._channelCount) : undefined;
+
+      const chunks = Array.from({ length: this._channelCount }).map(() => Array.from({ length: totalChunks }) as Float32Array[]);
 
       info('decode:chunk:start', this.src, chunkIndex, totalChunks);
 
@@ -183,7 +208,23 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
           if (this.sourceDecodeCancelled) return;
 
           if (value) {
-            chunks[chunkIndex] = value;
+            // Only 1 channel, just copy the data of the chunk directly
+            if (this._channelCount === 1) {
+              chunks[0][chunkIndex] = value;
+            } else {
+
+              if (!splitChannels) throw new Error('AudioDecoder: splitChannels not initialized');
+
+              // Multiple channels, split the data into separate channels within a web worker
+              // This is done to avoid blocking the UI thread
+              const channels = await splitChannels.split(value);
+
+              if (this.sourceDecodeCancelled) return;
+
+              channels.forEach((channel, index) => {
+                chunks[index][chunkIndex] = channel;
+              });
+            }
           }
 
           this.invoke('progress', [chunkIndex + 1, totalChunks]);
@@ -202,6 +243,7 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
 
       info('decode:complete', this.src);
     } finally {
+      splitChannels?.destroy();
       this.disposeWorker();
     }
   }
@@ -238,7 +280,7 @@ export class AudioDecoder extends Events<AudioDecoderEvents> {
       yield new Promise((resolve, reject) => {
         if (!this.worker || this.sourceDecodeCancelled) return resolve(null);
 
-        const nextChunkDuration = clamp(totalDuration - durationOffset, 0, DURATION_CHUNK_SIZE);
+        const nextChunkDuration = clamp(totalDuration - durationOffset, 0, this.getChunkDuration());
         const currentOffset = durationOffset;
 
         durationOffset += nextChunkDuration;