diff --git a/CMakeLists.txt b/CMakeLists.txt index a1d8d7c..6cf1a16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,21 +634,12 @@ if(EXISTS "${AI_MUSIC_GENERATION_SCRIPT}") ) endif() -set(AI_OPENSTUDIO_ACE_RUNNER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_runner.py") -if(EXISTS "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}") +set(AI_STABLE_AUDIO_GENERATION_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/stable_audio3_generate.py") +if(EXISTS "${AI_STABLE_AUDIO_GENERATION_SCRIPT}") openstudio_copy_runtime_file( - "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}" - "scripts/openstudio_ace_runner.py" - "Copying OpenStudio ACE runner script to output directory" - ) -endif() - -set(AI_OPENSTUDIO_ACE_BACKEND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_backend") -if(EXISTS "${AI_OPENSTUDIO_ACE_BACKEND_DIR}") - openstudio_copy_runtime_directory( - "${AI_OPENSTUDIO_ACE_BACKEND_DIR}" - "scripts/openstudio_ace_backend" - "Copying OpenStudio ACE backend package to output directory" + "${AI_STABLE_AUDIO_GENERATION_SCRIPT}" + "scripts/stable_audio3_generate.py" + "Copying Stable Audio 3 generation helper script to output directory" ) endif() @@ -708,6 +699,14 @@ openstudio_remove_runtime_file( "scripts/openstudio_ace_backend/parity_debug.py" "Removing ACE-Step parity backend helper from output directory" ) +openstudio_remove_runtime_file( + "scripts/openstudio_ace_runner.py" + "Removing legacy OpenStudio ACE split runner from output directory" +) +openstudio_remove_runtime_path( + "scripts/openstudio_ace_backend" + "Removing legacy OpenStudio ACE split backend from output directory" +) set(LEGACY_EXTERNAL_ACE_RUNNER_PATH "scripts/co") string(APPEND LEGACY_EXTERNAL_ACE_RUNNER_PATH "mfy_ace_runner.py") openstudio_remove_runtime_file( diff --git a/README.md b/README.md index 46f23ea..8e57262 100644 --- a/README.md +++ b/README.md @@ -1,398 +1,259 @@

- -

OpenStudio

+ OpenStudio logo +

+ +

OpenStudio

+ +

+ A native, open DAW for recording, editing, mixing, pitch work, plugin hosting, and local AI-assisted music production.

- A professional-grade DAW built with a JUCE C++ audio engine and a React/TypeScript UI — shipping natively on Windows and macOS. + Built with a JUCE C++ audio engine and a React/TypeScript interface. No Electron shell, no cloud-only workflow, no locked session model.

Platform: Windows and macOS JUCE 8 - React + TypeScript + React and TypeScript Audio drivers Plugin formats + AI music models

-### Project and Preset Formats -- Project files save as `.osproj` and continue opening legacy `.s13` sessions -- Theme exports use `.ostheme` -- Built-in FX presets use `.ospreset` -- Peak cache files use `.ospeaks` +

+ Quick Start + | + Features + | + Comparison + | + User Manual + | + Lua API +

-### Release Packaging -- Windows ships as a standard installer -- macOS ships as a free degraded unsigned DMG for v1 -- Automatic update metadata is served from `https://openstudio.org.in` -- Optional AI stem-separation tools install later from inside the app and are not bundled in the base download +> Screenshot/GIF slot: add a first-run timeline or mixer capture here once the current UI pass is ready. --- -## macOS installation debugging -- This command needs to be ran on macOS after installation through installer to un-quarantine the app and use it -- Otherwise the app would be shown as damaged or broken in macOS -```bash -xattr -dr com.apple.quarantine /Applications/OpenStudio.app -``` +## Why OpenStudio? -## What is OpenStudio? +OpenStudio is for people who want a real DAW surface, not just a prompt box, and a modern production workflow without giving up native audio performance. It combines classic multitrack production with newer tools such as stem separation, audio-to-MIDI, source-conditioned AI generation, graphical pitch editing, and scriptable project operations. -**OpenStudio** is a hybrid DAW with a high-performance C++ audio engine and a fully hardware-accelerated React frontend rendered inside a native desktop application. It combines the audio processing power of JUCE with the flexibility and speed of modern web UI tooling — no Electron, no Chromium overhead, just a native embedded web UI on top of the audio engine. +The goal is direct: record or import material, edit it deeply, route and mix it like a DAW, generate or transform ideas with local AI tools when useful, then render deliverables without leaving the session. - - +## Features At A Glance ---- +| Area | Current support | +|---|---| +| Recording | Multitrack audio recording, MIDI recording, input monitoring, punch range, record modes | +| Editing | Clip move/trim/split, ripple edit, razor edit, fades, grouping, takes, slip edit, reverse, normalize, time stretch, pitch shift | +| MIDI | MIDI tracks, instrument tracks, piano roll, velocity/CC editing, transforms, virtual keyboard, MIDI import/export, audio-to-MIDI | +| Mixing | Mixer, detached mixer, sends, buses, routing matrix, master strip, meter isolation, mixer snapshots | +| Plugins | VST3, CLAP, LV2 code paths, native plugin editors, input/track/master/monitoring FX, plugin presets, A/B states, MIDI learn | +| Built-in FX | EQ, compressor, gate, limiter, delay, reverb, chorus, saturator, pitch corrector, basic synth/piano/drums | +| Pitch | Graphical pitch editor, YIN analysis, note blobs, drift/vibrato tools, real-time auto-tune style FX, Basic Pitch polyphonic detection | +| AI | Optional AI Tools runtime, stem separation, ACE-Step music generation, Stable Audio 3 text-to-audio, clip variation, inpaint, continuation | +| Render | WAV, AIFF, FLAC, MP3, OGG, mono/stereo, sample-rate conversion, normalize, tail, dither, secondary output, stems, regions, razor areas | +| Delivery | Render queue, region render matrix, DDP export, batch converter, session archive, project compare, clean project directory | +| Workflow | Command palette, shortcut reference and rebinding, help overlay, getting started guide, screensets, themes, toolbar editor | +| Extensibility | Lua scripting API, script editor, JSFX/S13FX script effects, project automation helpers | +| Sync / media | Timecode display/settings, big clock, media explorer, media pool, missing media resolver, video window plumbing | -## Architecture +Some advanced features are still evolving. See [Implemented But Partial / Caveated](docs/implemented_features.md#implemented-but-partial--caveated) for the honest edges. -``` -C++ (JUCE) Backend React/TypeScript Frontend -┌─────────────────────────┐ ┌──────────────────────────────┐ -│ AudioEngine │◄─────►│ NativeBridge.ts │ -│ PlaybackEngine │ │ (window.__JUCE__ bridge) │ -│ AudioRecorder │ ├──────────────────────────────┤ -│ TrackProcessor │ │ useDAWStore (Zustand) │ -│ PluginManager │ ├──────────────────────────────┤ -│ MIDIManager │ │ Timeline (Konva canvas) │ -│ Metronome │ │ MixerPanel / ChannelStrip │ -│ PitchAnalyzer │ │ PitchEditorLowerZone │ -│ PitchResynthesizer │ │ PianoRoll / FXChainPanel │ -│ Pitch preview/scrub │ │ TransportBar / MenuBar │ -│ PitchCorrector │ │ PluginBrowser / RenderModal │ -│ StemSeparator │ └──────────────────────────────┘ -│ ARAHostController │ -└─────────────────────────┘ -``` +## Comparison -The **C++ backend** handles all audio I/O, recording, plugin hosting, MIDI device management, metering, pitch correction, and offline rendering. The **React frontend** handles all UI, state, canvas-based timeline, keyboard shortcuts, drag-and-drop, and project save/load. They communicate via a synchronous bridge exposed through `window.__JUCE__`. +These comparisons are meant to position OpenStudio clearly, not pretend a young open DAW replaces decades of commercial engineering in every studio tomorrow. ---- +### OpenStudio vs Cubase -## Flow Charts - -### Audio Signal Chain (Input → Output) - -```mermaid -flowchart LR - subgraph Input - MIC[Microphone / Line In\nASIO · WASAPI · DirectSound] - MIDI[MIDI Devices] - CLIP[Audio Clips on Disk] - end - - subgraph Per-Track Processing - direction TB - REC[AudioRecorder\nThreadedWriter] - PB[PlaybackEngine\nSample-rate-aware mixing] - IFX[Input FX Chain\nVST3 · CLAP · LV2 · JSFX] - TP[TrackProcessor\nVolume · Pan · Meter] - TFX[Track FX Chain\nVST3 · CLAP · LV2 · JSFX] - PITCH[PitchCorrector\nReal-time auto-tune] - end - - subgraph Master - MFX[Master FX Chain] - MIX[Master Mix] - end - - subgraph Output - DAC[Audio Output\nDAC / Speakers] - RENDER[Offline Render\nWAV · AIFF · FLAC] - end - - MIC --> REC --> IFX - CLIP --> PB --> IFX - MIDI --> TP - IFX --> PITCH --> TP --> TFX --> MFX --> MIX - MIX --> DAC - MIX --> RENDER -``` +| Need | Cubase | OpenStudio | +|---|---|---| +| Composition DAW | Mature commercial DAW with deep MIDI, scoring, VariAudio, built-in instruments, and large content libraries | Native open DAW with MIDI, instrument tracks, piano roll, audio-to-MIDI, pitch editor, and scriptable workflows | +| Pitch editing | VariAudio and ARA-compatible workflows | Built-in graphical pitch editor, real-time pitch corrector FX, Basic Pitch detection, and ARA hosting | +| AI assistance | Cubase Pro includes modern assisted tools such as stem separation | Optional local AI Tools runtime for stem separation plus text-to-music, text-to-audio, variation, inpaint, and continuation workflows | +| Extensibility | VST ecosystem, commercial extension model | VST3/CLAP/LV2, JSFX/S13FX, Lua API, open source codebase | +| Best fit | Established production/composition rooms that need Steinberg's mature ecosystem | Builders, indie producers, researchers, and power users who want DAW depth plus local AI and hackability | ---- +### OpenStudio vs Pro Tools -### Pitch Correction Pipeline - -```mermaid -flowchart TD - AUDIO[Original Clip Audio] - - subgraph Monophonic Path - direction TB - YIN[PitchAnalyzer\nYIN Detection · Note Segmentation] - NOTES[PitchNotes\npitch · formant · vibrato · drift per note] - VSF[Native VSF HQ\npitch-only note render] - PREVIEW[Signalsmith Stretch\nlive scrub/fast preview] - end - - subgraph Polyphonic Path - direction TB - POLY[PolyPitchDetector\nBasic-Pitch ONNX · 22050 Hz · 256 hop] - MASKS[HarmonicMaskGenerator\nWiener soft masks per note] - SPECSHIFT[SpectralPitchShifter\nper-note masked STFT shift] - end - - subgraph Real-time Path - RTFX[PitchCorrector\nper-block · key/scale-aware\nSignalsmithStretch presetCheaper] - end - - AUDIO --> YIN --> NOTES --> BACKEND --> RB --> OUT[Corrected Audio File\nreplaceClipAudioFile] - BACKEND --> NATIVE - BACKEND --> SS --> SPEAKERS - AUDIO --> POLY --> MASKS --> SPECSHIFT --> OUT - AUDIO --> RTFX --> SPEAKERS[Live Output] -``` +| Need | Pro Tools | OpenStudio | +|---|---|---| +| Studio standard | Mature industry-standard audio/post platform with AAX, HDX, large facility workflows, and advanced immersive/post tooling | Open native DAW focused on fast iteration, local-first production, and transparent implementation | +| Editing | Deep editing, recording, comping, post-production workflows | Multitrack recording, takes, razor/ripple editing, slip edit, clip properties, freeze, region tools | +| Pitch / repair | Strong third-party ecosystem through ARA partners and plugins | Built-in graphical pitch editor, real-time pitch corrector, ARA host controller, source-aware pitch render diagnostics | +| Delivery | Strong broadcast/post pipelines and collaboration tooling | Offline render, stems, region render matrix, render queue, DDP export, archive tools | +| Best fit | Commercial facilities and sessions that need Pro Tools compatibility | Producers and developers who want open workflows, modern UI, and integrated AI generation inside a DAW | ---- +### OpenStudio vs Suno AI -### Frontend Component Tree - -```mermaid -flowchart TD - APP[App.tsx] - - APP --> MB[MenuBar] - APP --> MT[MainToolbar\ntransport · snap · undo/redo] - APP --> WS[Workspace] - APP --> TB[TransportBar\nBPM · timecode · loop · metronome] - APP --> LZ[LowerZone] - APP --> MOD[Modals] - - WS --> TCP[Track Control Panel\nTrackHeader · SortableTrackHeader] - WS --> TL[Timeline\nKonva canvas · waveforms · clips · playhead] - - LZ --> PE[PitchEditorLowerZone\nPitchEditorCanvas · tools · viewport] - LZ --> CP[ClipPropertiesPanel] - LZ --> PR[PianoRoll\nKonva canvas] - - MOD --> SM[SettingsModal\naudio device config] - MOD --> RM[RenderModal\nexport settings] - MOD --> FX[FXChainPanel\nplugin browser · input FX · track FX] - MOD --> MX[MixerPanel\nChannelStrip · faders · meters] - MOD --> CL[ClipLauncherView\nAbleton-style grid] - MOD --> CMD[CommandPalette\nCtrl+Shift+P] - - subgraph State - STORE[useDAWStore\nZustand · ~3400 lines] - PESTORE[pitchEditorStore\nnotes · viewport · undo] - NB[NativeBridge.ts\nwindow.__JUCE__ calls] - end - - APP -.-> STORE - PE -.-> PESTORE - STORE -.-> NB - NB -.->|bridge| CPP[C++ AudioEngine] -``` +| Need | Suno AI | OpenStudio | +|---|---|---| +| Product shape | Cloud AI song generation service | Full DAW with optional local AI-assisted generation | +| Starting point | Prompt to generated song | Record, import, generate, edit, arrange, mix, render | +| Control | Fast idea generation with plan/credit limits and cloud queues | Timeline-level control over clips, tracks, routing, plugins, MIDI, pitch, stems, and exports | +| AI workflows | Song generation, editing, uploads, stems depending on plan | ACE-Step text-to-music and lyrics+style, Stable Audio 3 text-to-audio, source variation, inpaint, continuation, stem separation | +| Ownership/privacy model | Depends on Suno plan and terms; free plan is non-commercial as of the current Suno pricing/help pages | Local project files and local optional AI tooling; users decide what media enters the session | +| Best fit | Quickly making complete AI songs | Turning recordings, generated ideas, stems, and MIDI into editable productions | ---- +Reference pages used for the comparison: [Cubase editions](https://www.steinberg.net/cubase/compare-editions/), [Pro Tools](https://www.avid.com/en/pro-tools), [Suno pricing](https://suno.com/pricing), and [Suno rights help](https://help.suno.com/en/categories/550145-rights-ownership). -### JS ↔ C++ Bridge Data Flow - -```mermaid -sequenceDiagram - participant UI as React UI - participant NB as NativeBridge.ts - participant MC as MainComponent.cpp - participant AE as AudioEngine - - UI->>NB: addTrack(id, name, type) - NB->>MC: window.__JUCE__.backend.addTrack(json) - MC->>AE: audioEngine.addTrack(id, type) - AE-->>MC: ok - MC-->>NB: return "ok" - NB-->>UI: Promise resolved - - UI->>NB: applyPitchCorrection(trackId, clipId, notes) - NB->>MC: backend.applyPitchCorrection(json) - MC->>AE: applyPitchCorrection(...) - AE->>AE: native VSF note-HQ pitch-only render - AE->>AE: replaceClipAudioFile(newPath) - AE-->>MC: newFilePath - MC-->>UI: newFilePath (React re-renders waveform) - - loop 60fps playback - AE->>MC: meterLevels via event - MC->>UI: dispatchEvent("meterUpdate") - UI->>UI: setMeterLevels() — isolated store slice - end -``` +## AI Music And Assisted Audio ---- +OpenStudio keeps AI as part of the DAW workflow rather than a replacement for it. -## Features - -### Audio Engine -- **ASIO / WASAPI / DirectSound** driver support -- Multi-track audio recording with thread-safe `ThreadedWriter` -- Sample-rate-aware clip playback with linear interpolation -- Per-track **Input FX chain** + **Track FX chain** + **Master FX chain** -- PRO DAW-inspired **multi-resolution peak cache** (`.ospeaks` sidecar files, with legacy `.s13peaks` support) -- Offline render/export to WAV, AIFF, FLAC at 16/24/32-bit - -### Plugin Hosting -- **VST3**, **CLAP**, and **LV2** plugin formats -- Native plugin editor window management -- ARA plugin hosting (e.g. Melodyne, RePitch) -- JSFX / Lua script-based processors with `@gfx` rendering -- Built-in effects: EQ, compressor, stereo tool, delay, reverb, and more - -### Pitch Editor -- **Graphical pitch editor** (Melodyne/VariAudio-style) — analyze, display, and redraw pitch curves per note -- **Real-time auto-tune** corrector (built-in pitch corrector FX) — key/scale-aware, inserted as an FX plugin -- **Native VSF pitch apply** for graphical note-HQ pitch-only edits; Signalsmith Stretch remains limited to live scrub/fast preview and the realtime pitch-corrector FX. -- **Polyphonic pitch detection** via Spotify's Basic-Pitch ONNX model -- Formant shift, vibrato, drift, and transition controls per note - -### MIDI -- MIDI device enumeration and input routing -- Piano Roll editor with Konva canvas -- Virtual 88-key on-screen keyboard -- Per-track MIDI clip storage with time-range queries -- Metronome with BPM, time signature, and accent patterns - -### Timeline & UI -- Konva canvas-based timeline: waveforms, clips, rulers, zoom, drag, multi-selection -- Mixer panel with channel strips, faders, pan, solo/mute, metering -- AI stem separation (vocals / drums / bass / other) via optional in-app AI Tools install -- Automation lanes with envelope editing -- Clip launcher (Ableton-style grid) -- Full undo/redo via CommandManager for all data-modifying actions -- Command palette (`Ctrl+Shift+P`), keyboard shortcuts modal, action registry -- PRO DAW theme import/export (`.PRO DAWTheme` → `.ostheme`, with legacy `.s13theme` import support) +| Workflow | What it does | +|---|---| +| Text to Music | Generates a fresh music clip through ACE-Step from prompt, lyrics, BPM, key/scale, language, duration, seed, and diffusion controls | +| Lyrics + Style | Uses structured lyrics plus style/arrangement prompt for song generation | +| Text to Audio | Uses Stable Audio 3 Medium for prompt-based audio generation | +| Create Variation | Builds a source-conditioned variation while preserving the selected clip's identity | +| Inpaint Selection | Regenerates a selected range inside a clip while matching the surrounding audio | +| Continue Clip | Generates a continuation tail from the selected source clip | +| Stem Separation | Splits source audio into vocals, drums, bass, and other stems for remixing or cleanup | +| Audio to MIDI | Extracts MIDI from audio with Basic Pitch / ONNX plumbing where available | ---- +The base app does not bundle heavy AI runtimes. Optional AI Tools are installed from inside OpenStudio so the core DAW can stay lean. -## Built-In Effects +## DAW Workflow -| Effect | Description | -|--------|-------------| -| EQ | Per-channel graphical equalizer | -| Compressor | Dynamic range compression | -| Reverb | Algorithmic reverb | -| Delay | Stereo delay with feedback and filtering | -| Stereo Tool | Volume, pan, stereo width | -| Crusher | Bit-depth and sample-rate reduction | -| Pitch Corrector | Real-time key/scale-aware auto-tune | -| JSFX Processor | PRO DAW-compatible JSFX / Lua scripts | +### Record ---- +- ASIO, WASAPI, and DirectSound device support on Windows. +- Track arming, input selection, input monitoring, punch range, and record modes. +- Audio clips and MIDI clips land directly on the timeline. + +### Edit + +- Konva-powered arrange view with waveforms, MIDI thumbnails, grid, rulers, time selection, razor areas, and zoom/scroll navigation. +- Clip splitting, trimming, moving, copying, muting, grouping, locking, color, fades, auto-crossfade, reverse, normalize, time stretch, and pitch shift. +- Piano roll for note editing, velocity, CC lanes, pitch bend controls, quantize, scale snap, transpose, reverse, invert, and humanize. + +### Mix -## Build & Run +- Horizontal mixer with channel strips, master strip, detached mixer window, metering, snapshots, routing controls, sends, buses, phase invert, stereo width, sidechain routing, and monitoring FX. +- Built-in effects and third-party plugins can be used on input FX, track FX, master FX, and monitoring FX chains. + +### Tune + +- Graphical pitch editor for vocal-style correction with note blobs, contour editing, pitch/drift/vibrato/transition tools, correct-pitch macro, and pitch-editor undo/redo. +- Real-time pitch corrector FX for auto-tune style workflows. +- ARA hosting path for compatible pitch/audio editors. + +### Deliver + +- Render master mixes, selected-track stems, all stems, selected items, selected items through master, razor edit areas, project regions, or selected regions. +- Export WAV, AIFF, FLAC, MP3, and OGG with mono/stereo, target sample rate, bit depth/quality, normalize, render tail, dither, and secondary output. +- Use render queue, region render matrix, DDP export, batch converter, MIDI export, archive, project compare, and clean project tools for handoff. + +## Project And File Formats + +- Project files: `.osproj` +- Legacy project import/open: `.s13` +- Theme exports: `.ostheme` +- Legacy theme import: `.s13theme` +- Built-in FX presets: `.ospreset` +- Waveform peak cache: `.ospeaks` +- Legacy peak cache support: `.s13peaks` + +## Architecture + +```text +C++ / JUCE backend React / TypeScript frontend +-------------------------------- -------------------------------- +AudioEngine App.tsx +PlaybackEngine useDAWStore / actionRegistry +AudioRecorder NativeBridge.ts +TrackProcessor Timeline / PianoRoll / Mixer +PluginManager Render / FX / Routing panels +MIDIManager Pitch editor stores and canvas +PitchAnalyzer / PitchResynthesizer Help, shortcuts, command palette +StemSeparator / AITrackEngine AI workflow modals +ARAHostController Theme, scripts, project tools +``` + +The backend owns audio I/O, recording, playback, plugin processing, MIDI, metering, rendering, pitch/audio analysis, stem separation, and AI worker orchestration. The frontend owns layout, state, timeline interaction, editor panels, keyboard shortcuts, project serialization, and user workflows. The bridge is exposed through `window.__JUCE__.backend.*` and wrapped by `frontend/src/services/NativeBridge.ts`. + +## Quick Start ### Requirements -- **Windows 10/11** (x64) for the full local toolchain documented below -- **macOS** release packaging is supported in CI/release tooling, with unsigned DMG distribution for v1 -- [CMake](https://cmake.org/) ≥ 3.22 -- [Visual Studio 2022](https://visualstudio.microsoft.com/) with C++ workload -- [Node.js](https://nodejs.org/) ≥ 18 -- [Python](https://python.org/) ≥ 3.10 -- [NASM](https://nasm.us/) (required for YSFX) — add to `PATH` +- Windows 10/11 x64 for the primary local development flow +- macOS release packaging support for app/DMG builds +- CMake 3.22 or newer +- Visual Studio 2022 with C++ workload on Windows +- Node.js 18 or newer +- Python 3.10 or newer +- NASM in `PATH` for YSFX builds +- ARA SDK and supported third-party SDK/runtime assets as documented in the repo -### Quick Start +### Development ```bash -# Clone the repo -git clone https://github.com/your-org/studio13-v3.git && cd studio13-v3 - -# Full dev build: installs deps, builds C++ Debug, starts Vite HMR, launches app +# Full dev flow: installs deps, builds C++ Debug, starts Vite, launches app python build.py dev --run ``` -### Partial Rebuilds +Partial rebuilds: ```bash -# Frontend only — no C++ rebuild needed for UI changes +# Frontend only cd frontend && npm run dev -# C++ only — Debug +# C++ Debug cmake --build build --config Debug -# C++ only — Release +# C++ Release cmake --build build --config Release -# Production — embedded frontend + Release C++ → single .exe +# Production package path python build.py prod ``` ---- +## macOS First Launch Note -## Tech Stack +The v1 macOS community package is unsigned. If macOS reports that the app is damaged or blocked after installing from the DMG, remove quarantine: -### C++ Backend - -| Library | Purpose | -|---------|---------| -| [JUCE 8](https://juce.com/) | Audio engine, plugin hosting, WebBrowserComponent | -| [Signalsmith Stretch](https://github.com/Signalsmith-Audio/signalsmith-stretch) | MIT — pitch preview/scrub and realtime pitch-corrector FX | -| [YSFX](https://github.com/jpcima/ysfx) | JSFX / Lua script processor | -| [ONNX Runtime](https://onnxruntime.ai/) | Basic-Pitch polyphonic pitch detection | -| [sol2](https://github.com/ThePhD/sol2) | Lua scripting engine bindings | -| [WebView2](https://developer.microsoft.com/en-us/microsoft-edge/webview2/) | Embedded browser for the Windows React UI | -| ASIO SDK | Low-latency audio I/O | - -### React Frontend - -| Library | Purpose | -|---------|---------| -| [React 18](https://react.dev/) + TypeScript | UI framework | -| [Zustand](https://github.com/pmndrs/zustand) | Global state management | -| [react-konva](https://konvajs.org/) | Canvas-based timeline and piano roll | -| [Tailwind CSS v4](https://tailwindcss.com/) | Styling with custom `daw-*` color tokens | -| [@dnd-kit](https://dndkit.com/) | Drag-and-drop track reordering | -| [Vite](https://vitejs.dev/) | Dev server + production bundler | +```bash +xattr -dr com.apple.quarantine /Applications/OpenStudio.app +``` ---- +Then open the app from Finder, or right-click and choose **Open** if Gatekeeper asks for confirmation. -## Project Structure +## Documentation -``` -repo-root/ -├── Source/ # C++ audio engine, plugin hosting, pitch pipeline -├── frontend/ # React/TypeScript UI (Vite, Tailwind, Zustand, Konva) -├── resources/ # ONNX models, presets, icons -├── tools/ # ffmpeg.exe, stem separator, setup scripts -├── build/ # CMake output (generated) -├── CMakeLists.txt # C++ build config (JUCE, ASIO, WebView2, VST3, ONNX) -└── build.py # Orchestrator: cmake + npm + Vite dev server -``` +- [User Manual](docs/USER_MANUAL.md) +- [Lua Scripting API](docs/API.md) +- [Runtime Dependency Contract](docs/runtime-dependency-contract.md) +- [Release Runbook](docs/release-runbook.md) +- [Release Smoke Checklist](docs/release-smoke-checklist.md) +- [Implemented Features Audit](docs/implemented_features.md) ---- +Historical planning documents remain in `docs/` for traceability, but the codebase and `docs/implemented_features.md` should be treated as the fresher feature inventory. -## Roadmap - -### In Progress -- [ ] Graphical pitch editor — Melodyne/VariAudio parity (vibrato tool, multi-note operations) -- [ ] Polyphonic pitch correction rewrite -- [ ] ARA2 deep integration - -### Planned -- [ ] MIDI 2.0 support -- [ ] Score / notation view -- [ ] Linux release path -- [ ] Plugin sandboxing - -### Completed -- [x] VST3 / CLAP / LV2 plugin hosting -- [x] ARA plugin hosting controller -- [x] Signalsmith Stretch preview/scrub and realtime pitch-corrector support -- [x] Real-time auto-tune pitch corrector -- [x] Polyphonic pitch detection (Basic-Pitch ONNX) -- [x] PRO DAW-style multi-resolution peak cache -- [x] Optional AI stem separation install flow -- [x] JSFX / Lua `@gfx` rendering -- [x] PRO DAW theme import/export -- [x] Offline render (WAV / AIFF / FLAC, 16/24/32-bit) -- [x] Clip launcher (Ableton-style grid) -- [x] Automation lanes -- [x] Full undo/redo for all data-modifying actions -- [x] Command palette +## Tech Stack ---- +| Layer | Tools | +|---|---| +| Native audio | JUCE 8, ASIO SDK, WASAPI, DirectSound | +| Embedded UI | JUCE WebBrowserComponent, WebView2 on Windows | +| Frontend | React 18, TypeScript, Vite, Zustand, Tailwind CSS, Konva | +| Plugins | JUCE plugin hosting, VST3, CLAP integration, LV2 code paths, ARA SDK | +| Analysis / AI | ONNX Runtime, Basic Pitch model flow, ACE-Step, Stable Audio 3 optional runtime | +| Scripting | Lua / sol2, S13FX / JSFX-style script processor | +| Packaging | CMake, build.py orchestration, platform release scripts | + +## Status + +OpenStudio is under active development. Many core DAW workflows are implemented, while some professional edge cases remain experimental or partial: + +- Polyphonic correction/resynthesis is not at the same maturity as monophonic pitch analysis and editing. +- Some video, control-surface, live-capture, plugin-bridge, and interchange paths are still evolving. +- Heavy AI runtimes are optional and depend on local hardware, installed models, and runtime setup. +- Subjective audio quality claims, especially pitch/formant/stem artifacts, require listening tests. ## License -OpenStudio is currently distributed in this repository under AGPLv3-compatible terms. See `LICENSE` and `THIRD_PARTY_LICENSES.md` for the current licensing and dependency notices. +OpenStudio is distributed in this repository under AGPLv3-compatible terms. See `LICENSE` and [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md) for licensing and dependency notices. diff --git a/Source/AITrackEngine.cpp b/Source/AITrackEngine.cpp index 4299181..54b820f 100644 --- a/Source/AITrackEngine.cpp +++ b/Source/AITrackEngine.cpp @@ -2,7 +2,9 @@ namespace { -constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; +constexpr auto kPinnedMusicGenerationModelId = "ace-step-v15-xl-turbo"; +constexpr auto kPinnedMusicGenerationModelRepoId = "ACE-Step/acestep-v15-xl-turbo-diffusers"; +constexpr auto kStableAudioModelId = "stable-audio-3-medium"; constexpr auto kReaderSleepMs = 50; constexpr auto kWorkerStartupTimeoutMs = 45000; constexpr auto kWorkerRequestTimeoutMs = 10000; @@ -246,12 +248,24 @@ juce::File AITrackEngine::getUserRuntimeRoot() const return getUserDataRoot().getChildFile("stem-runtime"); } +juce::File AITrackEngine::getStableAudioRuntimeRoot() const +{ + return getUserDataRoot().getChildFile("stable-audio-runtime"); +} + juce::File AITrackEngine::getMusicGenerationCheckpointRoot() const { return juce::File::getSpecialLocation(juce::File::userHomeDirectory) .getChildFile(".cache") .getChildFile("ace-step") - .getChildFile("checkpoints"); + .getChildFile("diffusers"); +} + +juce::File AITrackEngine::getStableAudioModelRoot() const +{ + return getUserDataRoot() + .getChildFile("models") + .getChildFile(kStableAudioModelId); } juce::File AITrackEngine::findPython() const @@ -295,6 +309,15 @@ juce::File AITrackEngine::findPython() const return {}; } +juce::File AITrackEngine::findStableAudioPython() const +{ + auto runtimePython = findPythonInRuntimeRoot(getStableAudioRuntimeRoot()); + if (runtimePython.existsAsFile()) + return runtimePython; + + return {}; +} + juce::File AITrackEngine::findScript() const { const auto runtimeDir = getApplicationRuntimeDirectory(); @@ -315,6 +338,26 @@ juce::File AITrackEngine::findScript() const return {}; } +juce::File AITrackEngine::findStableAudioScript() const +{ + const auto runtimeDir = getApplicationRuntimeDirectory(); + const auto packagedScript = runtimeDir.getChildFile("scripts").getChildFile("stable_audio3_generate.py"); + if (packagedScript.existsAsFile()) + return packagedScript; + + const auto bundledDevScript = runtimeDir.getChildFile("tools").getChildFile("stable_audio3_generate.py"); + if (bundledDevScript.existsAsFile()) + return bundledDevScript; + + const auto workingCopyScript = juce::File::getCurrentWorkingDirectory() + .getChildFile("tools") + .getChildFile("stable_audio3_generate.py"); + if (workingCopyScript.existsAsFile()) + return workingCopyScript; + + return {}; +} + void AITrackEngine::cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const { #if JUCE_WINDOWS @@ -330,10 +373,14 @@ void AITrackEngine::cleanupLegacyWorkerProcesses(const juce::File& python, const "Get-CimInstance Win32_Process | " "Where-Object { $_.CommandLine -like '*--worker*' " " -and $_.CommandLine -like '*" + escapedScriptPath + "*' " - " -and $_.CommandLine -like '*" + escapedPythonPath + "*' } | " + " -and (" + " $_.CommandLine -like '*" + escapedPythonPath + "*' " + " -or $_.CommandLine -like '*stable_audio3_generate.py*' " + " -or $_.CommandLine -like '*generate_music.py*'" + " ) } | " "ForEach-Object { " " & taskkill /PID $_.ProcessId /F /T 2>$null | Out-Null; " - " Write-Output ('stopped legacy ACE-Step worker pid ' + $_.ProcessId) " + " Write-Output ('stopped legacy AI generation worker pid ' + $_.ProcessId) " "}"); juce::ChildProcess cleanup; @@ -415,9 +462,12 @@ bool AITrackEngine::waitForWorkerReady(int timeoutMs) if (currentProgress_.error.isEmpty()) { + const auto modelLabel = currentProgress_.modelId == kStableAudioModelId + ? juce::String("Stable Audio 3") + : juce::String("ACE-Step"); const auto message = workerExitedBeforeReady - ? appendProcessDetailsLocked("ACE-Step worker exited before reporting ready.") - : appendProcessDetailsLocked("ACE-Step worker did not become ready in time."); + ? appendProcessDetailsLocked(modelLabel + " worker exited before reporting ready.") + : appendProcessDetailsLocked(modelLabel + " worker did not become ready in time."); setProgressErrorLocked(workerExitedBeforeReady ? "worker_start_failed" : "worker_start_timeout", message, @@ -471,7 +521,7 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo if (workerPidToKill > 0) { juce::String killOutput; - juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process tree pid=" + juce::Logger::writeToLog("AITrackEngine: stopping AI generation child process tree pid=" + juce::String(workerPidToKill)); killedTree = killWindowsProcessTree(workerPidToKill, &killOutput); if (killOutput.isNotEmpty()) @@ -481,7 +531,7 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo if (! killedTree && processToKill->isRunning()) { - juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process"); + juce::Logger::writeToLog("AITrackEngine: stopping AI generation child process"); processToKill->kill(); } } @@ -501,8 +551,10 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo } } -bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce::File& script) +bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce::File& script, const juce::String& modelId) { + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); const auto expectedScriptVersion = computeScriptVersion(script); { const juce::ScopedLock sl(lock_); @@ -524,13 +576,23 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: command.add(python.getFullPathName()); command.add(script.getFullPathName()); command.add("--worker"); - command.add("--checkpoint-root"); - command.add(getMusicGenerationCheckpointRoot().getFullPathName()); - command.add("--music-gen-model"); - command.add(kPinnedMusicGenerationModelId); + if (isStableAudio) + { + command.add("--model-root"); + command.add(getStableAudioModelRoot().getFullPathName()); + } + else + { + command.add("--cache-root"); + command.add(getMusicGenerationCheckpointRoot().getFullPathName()); + command.add("--music-gen-model"); + command.add(kPinnedMusicGenerationModelId); + command.add("--model-id"); + command.add(kPinnedMusicGenerationModelRepoId); + } const auto commandLine = buildCommandLineForLog(command); - juce::Logger::writeToLog("AITrackEngine: launching persistent ACE-Step worker: " + commandLine + juce::Logger::writeToLog("AITrackEngine: launching persistent " + modelLabel + " worker: " + commandLine + " protocolVersion=" + juce::String(kWorkerProtocolVersion) + " expectedScriptVersion=" + expectedScriptVersion + " launchAttempt=" + juce::String(launchAttempt + 1)); @@ -550,7 +612,8 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: currentProgress_.state = "loading"; currentProgress_.progress = 0.02f; currentProgress_.phase = "starting_worker"; - currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.message = "Starting the " + modelLabel + " runtime session..."; + currentProgress_.modelId = modelId; currentProgress_.backend = "unknown"; currentProgress_.error.clear(); currentProgress_.runMode = "cold"; @@ -563,10 +626,13 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: currentProgress_.scriptVersion = expectedScriptVersion; currentProgress_.lastStdoutLine.clear(); currentProgress_.lastStderrLine.clear(); - currentProgress_.tracePath.clear(); - currentProgress_.failureDetail.clear(); + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.attemptMode.clear(); currentProgress_.lmBackend.clear(); currentProgress_.lmStage.clear(); + currentProgress_.tracePath.clear(); + currentProgress_.failureDetail.clear(); } if (! workerProcess_->start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) @@ -574,7 +640,7 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: const juce::ScopedLock sl(lock_); workerProcess_.reset(); setProgressErrorLocked("worker_start_failed", - "Failed to start the ACE-Step runtime session.", + "Failed to start the " + modelLabel + " runtime session.", "worker_start"); return false; } @@ -599,10 +665,13 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: return false; } -bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, +bool AITrackEngine::sendGenerateRequest(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile) { + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); int port = 0; juce::String requestId; { @@ -617,7 +686,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_connect_failed", - appendProcessDetailsLocked("ACE-Step worker did not provide a listening port."), + appendProcessDetailsLocked(modelLabel + " worker did not provide a listening port."), "worker_request"); return false; } @@ -627,13 +696,14 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_connect_failed", - appendProcessDetailsLocked("OpenStudio could not contact the ACE-Step runtime session."), + appendProcessDetailsLocked("OpenStudio could not contact the " + modelLabel + " runtime session."), "worker_request"); return false; } auto request = std::make_unique(); request->setProperty("command", "generate"); + request->setProperty("modelId", modelId); request->setProperty("workflow", workflowId); request->setProperty("params", paramsJson); request->setProperty("output", outputFile.getFullPathName()); @@ -649,7 +719,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, return false; setProgressErrorLocked("worker_request_failed", - appendProcessDetailsLocked("OpenStudio could not fully submit the generation request to the ACE-Step session."), + appendProcessDetailsLocked("OpenStudio could not fully submit the generation request to the " + modelLabel + " session."), "worker_protocol"); return false; } @@ -668,7 +738,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, return false; setProgressErrorLocked("worker_request_timeout", - appendProcessDetailsLocked("ACE-Step did not acknowledge the generation request in time."), + appendProcessDetailsLocked(modelLabel + " did not acknowledge the generation request in time."), "worker_protocol"); return false; } @@ -678,10 +748,10 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { auto error = object != nullptr ? object->getProperty("error").toString() - : "ACE-Step returned an invalid worker response."; + : modelLabel + " returned an invalid worker response."; if (error.isEmpty()) - error = "ACE-Step rejected the generation request."; + error = modelLabel + " rejected the generation request."; const juce::ScopedLock sl(lock_); if (cancelRequested_) @@ -704,7 +774,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_protocol_failed", - appendProcessDetailsLocked("ACE-Step acknowledged the request with a mismatched protocol or request id."), + appendProcessDetailsLocked(modelLabel + " acknowledged the request with a mismatched protocol or request id."), "worker_protocol"); return false; } @@ -718,11 +788,15 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, void AITrackEngine::launchGenerationTask(const juce::File& python, const juce::File& script, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile) { - if (! ensureWorkerAvailable(python, script)) + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); + + if (! ensureWorkerAvailable(python, script, modelId)) { const juce::ScopedLock sl(lock_); generationActive_ = false; @@ -749,7 +823,9 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, currentProgress_.state = "loading"; currentProgress_.progress = 0.03f; currentProgress_.phase = "submitting_request"; - currentProgress_.message = "Submitting the generation request to the ACE-Step session..."; + currentProgress_.message = "Submitting the generation request to the " + modelLabel + " session..."; + currentProgress_.modelId = modelId; + currentProgress_.workflowId = workflowId; currentProgress_.outputFile.clear(); currentProgress_.error.clear(); currentProgress_.elapsedMs = 0.0; @@ -759,9 +835,14 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, currentProgress_.sessionMode = "persistent"; currentProgress_.workerExitCode = 0; currentProgress_.failureKind.clear(); + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.attemptMode.clear(); + currentProgress_.lmBackend.clear(); + currentProgress_.lmStage.clear(); } - if (! sendGenerateRequest(workflowId, paramsJson, outputFile)) + if (! sendGenerateRequest(modelId, workflowId, paramsJson, outputFile)) { const juce::ScopedLock sl(lock_); generationActive_ = false; @@ -774,7 +855,8 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, } } -bool AITrackEngine::startGeneration(const juce::String& workflowId, +bool AITrackEngine::startGeneration(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputDir) { @@ -786,14 +868,16 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, return false; } - const auto python = findPython(); - const auto script = findScript(); + const auto isStableAudio = modelId == kStableAudioModelId; + const auto python = isStableAudio ? findStableAudioPython() : findPython(); + const auto script = isStableAudio ? findStableAudioScript() : findScript(); + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); if (! python.existsAsFile() || ! script.existsAsFile()) { const juce::ScopedLock sl(lock_); setProgressErrorLocked("runtime_missing", - "AI runtime is not ready. Install AI Tools first.", + modelLabel + " runtime is not ready. Install AI Tools first.", "runtime_missing"); return false; } @@ -811,7 +895,8 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); currentOutputFile_ = outputDir.getChildFile( - "generated_music_" + createSafeMusicGenerationTimestamp() + ".wav"); + (isStableAudio ? "generated_stable_audio_" : "generated_music_") + + createSafeMusicGenerationTimestamp() + ".wav"); outputFile = currentOutputFile_; generationActive_ = true; cancelRequested_ = false; @@ -823,7 +908,10 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, currentProgress_.state = "loading"; currentProgress_.progress = 0.01f; currentProgress_.phase = "starting"; - currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.message = "Starting the " + modelLabel + " runtime session..."; + currentProgress_.modelId = modelId; + currentProgress_.workflowId = workflowId; + currentProgress_.sourceClipId.clear(); currentProgress_.outputFile.clear(); currentProgress_.error.clear(); currentProgress_.elapsedMs = 0.0; @@ -836,18 +924,22 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, currentProgress_.lastStdoutLine.clear(); currentProgress_.lastStderrLine.clear(); currentProgress_.statusNote.clear(); - currentProgress_.attemptMode = "lm_dit"; + currentProgress_.attemptMode = isStableAudio ? juce::String() : juce::String("lm_dit"); currentProgress_.attemptIndex = 1; currentProgress_.protocolVersion = kWorkerProtocolVersion; currentProgress_.scriptVersion = expectedScriptVersion_; currentProgress_.requestId = currentRequestId_; currentProgress_.priorFailure.clear(); currentProgress_.lastProgressAgeMs = 0.0; + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.lmBackend.clear(); + currentProgress_.lmStage.clear(); } - generationThread_ = std::thread([this, python, script, workflowId, paramsJson, outputFile]() + generationThread_ = std::thread([this, python, script, modelId, workflowId, paramsJson, outputFile]() { - launchGenerationTask(python, script, workflowId, paramsJson, outputFile); + launchGenerationTask(python, script, modelId, workflowId, paramsJson, outputFile); }); return true; @@ -906,11 +998,14 @@ void AITrackEngine::parseOutputLine(const juce::String& line) if (workerProtocolVersion_ != kWorkerProtocolVersion || (! expectedScriptVersion_.isEmpty() && workerScriptVersion_ != expectedScriptVersion_)) { + const auto modelLabel = currentProgress_.modelId == kStableAudioModelId + ? juce::String("Stable Audio 3") + : juce::String("ACE-Step"); workerProtocolRejected_ = true; workerReady_ = false; workerPort_ = 0; setProgressErrorLocked("worker_protocol_failed", - appendProcessDetailsLocked("ACE-Step worker protocol or script version mismatch."), + appendProcessDetailsLocked(modelLabel + " worker protocol or script version mismatch."), "worker_protocol"); currentProgress_.statusNote = "Rejecting stale worker session before generation starts."; juce::Logger::writeToLog("AITrackEngine: rejecting worker ready handshake due to version mismatch" @@ -923,11 +1018,13 @@ void AITrackEngine::parseOutputLine(const juce::String& line) workerReady_ = true; workerPort_ = static_cast (double (obj->getProperty("port"))); + const auto backend = obj->getProperty("backend").toString(); + const auto readyLabel = backend == "stable-audio-3" ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); currentProgress_.state = "idle"; currentProgress_.progress = 0.0f; currentProgress_.phase = "worker_ready"; - currentProgress_.message = "ACE-Step runtime session is ready."; - currentProgress_.backend = obj->getProperty("backend").toString(); + currentProgress_.message = readyLabel + " runtime session is ready."; + currentProgress_.backend = backend; currentProgress_.sessionMode = obj->hasProperty("sessionMode") ? obj->getProperty("sessionMode").toString() : "persistent"; @@ -987,6 +1084,12 @@ void AITrackEngine::parseOutputLine(const juce::String& line) currentProgress_.backend = obj->getProperty("backend").toString(); if (obj->hasProperty("outputFile")) currentProgress_.outputFile = obj->getProperty("outputFile").toString(); + if (obj->hasProperty("modelId")) + currentProgress_.modelId = obj->getProperty("modelId").toString(); + if (obj->hasProperty("workflowId")) + currentProgress_.workflowId = obj->getProperty("workflowId").toString(); + if (obj->hasProperty("sourceClipId")) + currentProgress_.sourceClipId = obj->getProperty("sourceClipId").toString(); if (obj->hasProperty("error")) currentProgress_.error = obj->getProperty("error").toString(); if (obj->hasProperty("elapsedMs")) @@ -1265,6 +1368,7 @@ void AITrackEngine::stopWorker(bool clearProgress, bool userCancelled) AIGenerationProgress AITrackEngine::pollProgress() { bool shouldStopForDecodeStall = false; + bool shouldReleaseTerminalWorker = false; { const juce::ScopedLock sl(lock_); @@ -1324,10 +1428,23 @@ AIGenerationProgress AITrackEngine::pollProgress() currentProgress_.workerExitCode = workerExitCode_; currentProgress_.lastStdoutLine = lastStdoutLine_; currentProgress_.lastStderrLine = lastStderrLine_; + + const auto inTerminalState = currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled"; + if (inTerminalState + && ! generationActive_ + && workerProcess_ != nullptr + && workerProcess_->isRunning()) + { + shouldReleaseTerminalWorker = true; + } } if (shouldStopForDecodeStall) stopWorkerSession(false, false, false); + else if (shouldReleaseTerminalWorker) + stopWorkerSession(false, false, false); const juce::ScopedLock sl(lock_); return currentProgress_; diff --git a/Source/AITrackEngine.h b/Source/AITrackEngine.h index 05fcacb..6d935a4 100644 --- a/Source/AITrackEngine.h +++ b/Source/AITrackEngine.h @@ -14,6 +14,9 @@ struct AIGenerationProgress juce::String message; juce::String backend { "unknown" }; juce::String outputFile; + juce::String modelId; + juce::String workflowId; + juce::String sourceClipId; juce::String error; double elapsedMs = 0.0; double heartbeatTs = 0.0; @@ -47,7 +50,8 @@ class AITrackEngine AITrackEngine() = default; ~AITrackEngine(); - bool startGeneration(const juce::String& workflowId, + bool startGeneration(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputDir); @@ -58,16 +62,22 @@ class AITrackEngine private: juce::File getUserDataRoot() const; juce::File getUserRuntimeRoot() const; + juce::File getStableAudioRuntimeRoot() const; juce::File getMusicGenerationCheckpointRoot() const; + juce::File getStableAudioModelRoot() const; juce::File findPython() const; + juce::File findStableAudioPython() const; juce::File findScript() const; + juce::File findStableAudioScript() const; void cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const; - bool ensureWorkerAvailable(const juce::File& python, const juce::File& script); - bool sendGenerateRequest(const juce::String& workflowId, + bool ensureWorkerAvailable(const juce::File& python, const juce::File& script, const juce::String& modelId); + bool sendGenerateRequest(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile); void launchGenerationTask(const juce::File& python, const juce::File& script, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile); diff --git a/Source/AudioEngine.cpp b/Source/AudioEngine.cpp index 80da129..e9219b9 100644 --- a/Source/AudioEngine.cpp +++ b/Source/AudioEngine.cpp @@ -3920,10 +3920,13 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha if (track->hasActiveARA()) anyActiveARAInCallback = true; - // Solo logic: if any track is soloed, skip non-soloed tracks entirely - // (recording still works because record-armed tracks should also be soloed, - // and in practice users don't solo-off a track they're actively recording) - if (anySoloed && !track->getSolo()) + const bool excludedBySolo = anySoloed && !track->getSolo(); + const bool hasActiveAudioRecording = isRecordMode.load() && audioRecorder.isRecording(trackId); + const bool needsRecordingPath = track->getRecordArmed() || hasActiveAudioRecording; + + // Solo should silence non-soloed tracks, but it must not prevent an + // armed/active track from reaching the recorder. + if (excludedBySolo && !needsRecordingPath) { track->resetRMS(); continue; @@ -3938,7 +3941,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha && !staticMuteBlocksInput && (track->getRecordArmed() || track->getInputMonitoring() - || audioRecorder.isRecording(trackId)); + || hasActiveAudioRecording); const double blockStartTimeSeconds = currentSamplePosition / currentSampleRate; const bool shouldProcessMidi = track->needsProcessing(blockStartTimeSeconds, numSamples, currentSampleRate, isPlaying.load()); @@ -4049,7 +4052,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha // ========== RECORD RAW AUDIO (BEFORE FX) ========== // Write to recorder if transport is playing AND in record mode // This captures the raw input BEFORE any FX processing - if (isPlaying && isRecordMode && audioRecorder.isRecording(trackId)) + if (isPlaying && isRecordMode.load() && hasActiveAudioRecording) { // Punch recording: only write audio within punch range bool shouldWrite = true; @@ -4144,7 +4147,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha } // ========== SEND MIXING: fill destination track send accum buffers ========== - if (!trackEntry.sends.empty()) + if (!excludedBySolo && !trackEntry.sends.empty()) { const auto& preFaderBuffer = track->getPreFaderBuffer(); for (const auto& send : trackEntry.sends) @@ -4194,7 +4197,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha track->sendMIDIToOutput(midiMessages, currentSampleRate, track->getMute()); // Mix track output to device outputs (only if master send is enabled) - if (track->getMasterSendEnabled()) + if (!excludedBySolo && track->getMasterSendEnabled()) { int outStart = track->getOutputStartChannel(); int outCount = track->getOutputChannelCount(); @@ -6568,6 +6571,44 @@ juce::var AudioEngine::runAutomatedRegressionSuite() addSuite("plugin_capability_matrix", false, "Guardrail payload missing"); } + { + constexpr double fixtureSampleRate = 48000.0; + constexpr int fixtureBlockSize = 512; + constexpr double fixtureBpm = 120.0; + const double samplesPerBeat = (60.0 / fixtureBpm) * fixtureSampleRate; + + auto firstClickSampleForStart = [&] (double startSample) + { + Metronome probe; + probe.prepareToPlay(fixtureSampleRate, fixtureBlockSize); + probe.setBpm(fixtureBpm); + probe.setTimeSignature(4, 4); + probe.setVolume(1.0f); + probe.setEnabled(true); + juce::AudioBuffer buffer(2, fixtureBlockSize); + buffer.clear(); + probe.getNextAudioBlock(buffer, startSample); + for (int sample = 0; sample < buffer.getNumSamples(); ++sample) + { + if (std::abs(buffer.getSample(0, sample)) > 1.0e-5f + || std::abs(buffer.getSample(1, sample)) > 1.0e-5f) + return sample; + } + return -1; + }; + + const int sampleZeroClick = firstClickSampleForStart(0.0); + const int midBeatClick = firstClickSampleForStart(samplesPerBeat * 0.5); + const int boundaryClick = firstClickSampleForStart(samplesPerBeat); + const bool metronomeBoundaryPass = sampleZeroClick == 1 + && midBeatClick < 0 + && boundaryClick == 1; + addSuite("metronome_boundary_start_fixture", metronomeBoundaryPass, + "sample0FirstClick=" + juce::String(sampleZeroClick) + + ", midBeatFirstClick=" + juce::String(midBeatClick) + + ", boundaryFirstClick=" + juce::String(boundaryClick)); + } + auto bufferFiniteAndBounded = [] (const juce::AudioBuffer& buffer, float peakLimit) { for (int ch = 0; ch < buffer.getNumChannels(); ++ch) @@ -10749,6 +10790,25 @@ juce::var AudioEngine::getWaveformPeaks(const juce::String& filePath, int sample return peakCache.getPeaks(audioFile, samplesPerPixel, startSample, numPixels); } +bool AudioEngine::refreshWaveformPeaks(const juce::String& filePath) +{ + juce::File audioFile(filePath); + if (!audioFile.existsAsFile()) + { + juce::Logger::writeToLog("refreshWaveformPeaks: File not found: " + filePath); + return false; + } + + peakCache.invalidate(audioFile); + peakCache.generateAsync(audioFile, [this, filePath]() { + juce::Logger::writeToLog("refreshWaveformPeaks: Peak cache ready for: " + filePath); + if (onPeaksReady) + onPeaksReady(filePath); + }); + + return true; +} + juce::var AudioEngine::getRecordingPeaks(const juce::String& trackId, int samplesPerPixel, int numPixels) { logAudioRecord("getRecordingPeaks track=" + trackId @@ -11574,7 +11634,8 @@ juce::File AudioEngine::findFFmpegExe() const } bool AudioEngine::convertWithFFmpeg(const juce::File& inputFile, const juce::File& outputFile, - const juce::String& format, double targetSampleRate, int quality) const + const juce::String& format, double targetSampleRate, int quality, + int bitDepth, int numChannels) const { auto ffmpeg = findFFmpegExe(); if (!ffmpeg.existsAsFile()) @@ -11589,15 +11650,32 @@ bool AudioEngine::convertWithFFmpeg(const juce::File& inputFile, const juce::Fil args.add("-y"); // Overwrite output args.add("-i"); args.add(inputFile.getFullPathName()); // Input file + args.add("-map"); + args.add("0:a:0"); + args.add("-vn"); + args.add("-sn"); + args.add("-dn"); + + juce::String formatLower = format.toLowerCase(); + double outputSampleRate = targetSampleRate; + if (formatLower == "mp3" && outputSampleRate > 48000.0) + { + logToDisk("convertWithFFmpeg: MP3 does not support " + juce::String(outputSampleRate) + + " Hz; using 48000 Hz for MP3 output"); + outputSampleRate = 48000.0; + } // Sample rate conversion (if target differs from source) - if (targetSampleRate > 0) + if (outputSampleRate > 0) { args.add("-ar"); - args.add(juce::String((int)targetSampleRate)); + args.add(juce::String((int)outputSampleRate)); + } + if (numChannels == 1 || numChannels == 2) + { + args.add("-ac"); + args.add(juce::String(numChannels)); } - - juce::String formatLower = format.toLowerCase(); if (formatLower == "mp3") { @@ -11617,6 +11695,29 @@ bool AudioEngine::convertWithFFmpeg(const juce::File& inputFile, const juce::Fil args.add("-q:a"); args.add(juce::String(q)); } + else if (formatLower == "wav") + { + args.add("-codec:a"); + args.add(bitDepth == 32 ? "pcm_f32le" : (bitDepth == 16 ? "pcm_s16le" : "pcm_s24le")); + } + else if (formatLower == "aiff" || formatLower == "aif") + { + args.add("-codec:a"); + args.add(bitDepth == 32 ? "pcm_f32be" : (bitDepth == 16 ? "pcm_s16be" : "pcm_s24be")); + } + else if (formatLower == "flac") + { + const int flacBitDepth = bitDepth >= 24 ? 24 : 16; + args.add("-codec:a"); + args.add("flac"); + args.add("-sample_fmt"); + args.add(flacBitDepth == 24 ? "s32" : "s16"); + if (flacBitDepth == 24) + { + args.add("-bits_per_raw_sample"); + args.add("24"); + } + } else { // WAV/AIFF/FLAC sample rate conversion only — keep format as-is @@ -11690,8 +11791,9 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do // Use requested sample rate for render if provided, otherwise fall back to device rate. // PlaybackEngine::fillTrackBuffer() handles per-file SR conversion automatically, // so rendering at a different rate than the device is fully supported. - double actualSampleRate = (renderSampleRate > 0) ? renderSampleRate : currentSampleRate; - if (actualSampleRate <= 0) actualSampleRate = 44100.0; + double requestedOutputSampleRate = (renderSampleRate > 0) ? renderSampleRate : currentSampleRate; + if (requestedOutputSampleRate <= 0) requestedOutputSampleRate = 44100.0; + double actualSampleRate = requestedOutputSampleRate; if (bitDepth != 16 && bitDepth != 24 && bitDepth != 32) bitDepth = 24; if (numChannels < 1 || numChannels > 2) numChannels = 2; @@ -11713,8 +11815,13 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do if (isStemRender) stemTrackId = source.substring(5); // After "stem:" - logToDisk("renderProject: Using actualSampleRate=" + juce::String(actualSampleRate) + - (actualSampleRate != currentSampleRate ? " (differs from device rate " + juce::String(currentSampleRate) + ")" : " (device rate)")); + const bool canUseFFmpegSampleRatePostProcess = + isLossyFormat + || formatLower == "wav" + || formatLower == "aiff" + || formatLower == "aif" + || formatLower == "flac"; + if (isLossyFormat) logToDisk("renderProject: Lossy format=" + formatLower + " codecQuality=" + juce::String(codecQuality)); if (isStemRender) @@ -11872,6 +11979,67 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do logToDisk("renderProject: " + juce::String((int)trackSnapshots.size()) + " tracks, anySoloed=" + juce::String(anySoloed ? "true" : "false")); + if (canUseFFmpegSampleRatePostProcess) + { + double includedClipSampleRate = 0.0; + bool includedClipsHaveSingleSampleRate = false; + bool includedClipSampleRateMismatch = false; + juce::AudioFormatManager sourceRateFormatManager; + sourceRateFormatManager.registerBasicFormats(); + + for (const auto& clip : clipSnapshot) + { + if (! clip.isActive) + continue; + + auto trackIt = trackSnapshotById.find(clip.trackId); + if (trackIt == trackSnapshotById.end()) + continue; + + const auto& snap = trackIt->second; + if (isStemRender && snap.id != stemTrackId) + continue; + if (! isStemRender && snap.muted) + continue; + if (! isStemRender && anySoloed && ! snap.soloed) + continue; + + std::unique_ptr reader(sourceRateFormatManager.createReaderFor(clip.audioFile)); + if (! reader || reader->sampleRate <= 0.0) + continue; + + if (! includedClipsHaveSingleSampleRate) + { + includedClipSampleRate = reader->sampleRate; + includedClipsHaveSingleSampleRate = true; + } + else if (std::abs(includedClipSampleRate - reader->sampleRate) > 1.0) + { + includedClipSampleRateMismatch = true; + break; + } + } + + if (includedClipsHaveSingleSampleRate + && ! includedClipSampleRateMismatch + && std::abs(includedClipSampleRate - requestedOutputSampleRate) > 1.0 + && includedClipSampleRate <= 192000.0) + { + logToDisk("renderProject: intermediate render sample rate set from " + + juce::String(actualSampleRate) + " to source rate " + + juce::String(includedClipSampleRate) + + " so FFmpeg handles final SRC to " + + juce::String(requestedOutputSampleRate)); + actualSampleRate = includedClipSampleRate; + needsFFmpegPostProcess = true; + } + } + + logToDisk("renderProject: Using actualSampleRate=" + juce::String(actualSampleRate) + + (std::abs(actualSampleRate - requestedOutputSampleRate) > 1.0 + ? " (ffmpeg intermediate; requested output rate " + juce::String(requestedOutputSampleRate) + ")" + : (actualSampleRate != currentSampleRate ? " (differs from device rate " + juce::String(currentSampleRate) + ")" : " (device rate)"))); + struct ScopedTrackMuteBypass { TrackProcessor* track = nullptr; @@ -12208,7 +12376,10 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do root->setProperty("requestedOutputFile", outputFile.getFullPathName()); root->setProperty("renderFile", renderFile.getFullPathName()); root->setProperty("format", format); + root->setProperty("requestedOutputSampleRate", requestedOutputSampleRate); root->setProperty("actualSampleRate", actualSampleRate); + root->setProperty("ffmpegPostProcess", needsFFmpegPostProcess); + root->setProperty("lossyIntermediateRender", isLossyFormat); root->setProperty("bitDepth", bitDepth); root->setProperty("channels", numChannels); root->setProperty("normalize", normalize); @@ -12962,12 +13133,14 @@ bool AudioEngine::renderProject(const juce::String& source, double startTime, do logToDisk ("renderProject: signal-chain debug report=" + reportFile.getFullPathName()); } - // ========== 9. FFmpeg post-processing (lossy encoding only) ========== + // ========== 9. FFmpeg post-processing (encoding or final SRC) ========== if (needsFFmpegPostProcess) { - logToDisk("renderProject: Starting FFmpeg post-processing (lossy encoding)..."); - // SR conversion is already handled natively — no need to pass targetSR here - bool ffmpegOk = convertWithFFmpeg(renderFile, outputFile, formatLower, 0, codecQuality); + logToDisk("renderProject: Starting FFmpeg post-processing (encoding/final SRC)..."); + // Let FFmpeg perform the final output SRC/encode step. + bool ffmpegOk = convertWithFFmpeg(renderFile, outputFile, formatLower, + requestedOutputSampleRate, codecQuality, + bitDepth, numChannels); // Clean up temp file renderFile.deleteFile(); @@ -16541,30 +16714,41 @@ bool AudioEngine::isPolyphonicDetectionAvailable() const #endif } +static juce::var makePolyAnalysisError(const juce::String& clipId, const juce::String& error) +{ + auto errObj = std::make_unique(); + errObj->setProperty("error", error); + errObj->setProperty("clipId", clipId); + errObj->setProperty("sampleRate", 22050.0); + errObj->setProperty("hopSize", 256); + errObj->setProperty("pitchSalience", juce::Array()); + errObj->setProperty("salienceDownsampleFactor", 1); + errObj->setProperty("notes", juce::Array()); + return juce::var(errObj.release()); +} + juce::var AudioEngine::analyzePolyphonic(const juce::String& trackId, const juce::String& clipId) { - // Lazy-load the ONNX model on first use - if (! polyModelLoadAttempted) + // Lazy-load the ONNX model on first use. Basic Pitch/ONNX Runtime objects are + // shared by all conversion requests, so model load and inference are serialized. { - polyModelLoadAttempted = true; - auto exeDir = getApplicationRuntimeDirectory(); - auto modelFile = exeDir.getChildFile("models").getChildFile("basic_pitch_nmp.onnx"); - if (! modelFile.existsAsFile()) + const juce::ScopedLock lock(polyAnalysisLock); + if (! polyModelLoadAttempted) { - juce::Logger::writeToLog("analyzePolyphonic: Model not found at " + modelFile.getFullPathName()); - // Try alternative location - modelFile = exeDir.getChildFile("basic_pitch_nmp.onnx"); + polyModelLoadAttempted = true; + auto exeDir = getApplicationRuntimeDirectory(); + auto modelFile = exeDir.getChildFile("models").getChildFile("basic_pitch_nmp.onnx"); + if (! modelFile.existsAsFile()) + { + juce::Logger::writeToLog("analyzePolyphonic: Model not found at " + modelFile.getFullPathName()); + // Try alternative location + modelFile = exeDir.getChildFile("basic_pitch_nmp.onnx"); + } + polyPitchDetector.loadModel(modelFile); } - polyPitchDetector.loadModel(modelFile); - } - if (! polyPitchDetector.isModelLoaded()) - { - auto errObj = std::make_unique(); - errObj->setProperty("error", "Polyphonic model not loaded. Place basic_pitch_nmp.onnx in the models/ directory."); - errObj->setProperty("clipId", clipId); - errObj->setProperty("notes", juce::Array()); - return juce::var(errObj.release()); + if (! polyPitchDetector.isModelLoaded()) + return makePolyAnalysisError(clipId, "Polyphonic model not loaded. Place basic_pitch_nmp.onnx in the models/ directory."); } // Find the clip's audio file @@ -16579,23 +16763,33 @@ juce::var AudioEngine::analyzePolyphonic(const juce::String& trackId, const juce } } - if (! foundClip || ! foundClip->audioFile.existsAsFile()) - return juce::var(); + if (! foundClip) + return makePolyAnalysisError(clipId, "Audio clip not found for MIDI conversion."); + + if (! foundClip->audioFile.existsAsFile()) + return makePolyAnalysisError(clipId, "Audio file for clip is missing."); // Read audio file juce::AudioFormatManager fmtMgr; fmtMgr.registerBasicFormats(); std::unique_ptr reader(fmtMgr.createReaderFor(foundClip->audioFile)); - if (! reader) return juce::var(); + if (! reader) + return makePolyAnalysisError(clipId, "Could not read audio file for MIDI conversion."); - int startSample = static_cast(foundClip->offset * reader->sampleRate); - int numSamples = static_cast(foundClip->duration * reader->sampleRate); - numSamples = std::min(numSamples, static_cast(reader->lengthInSamples) - startSample); - if (numSamples <= 0) return juce::var(); + const int64_t startSample64 = static_cast(std::round(foundClip->offset * reader->sampleRate)); + const int64_t requestedSamples64 = static_cast(std::round(foundClip->duration * reader->sampleRate)); + if (startSample64 < 0 || requestedSamples64 <= 0 || startSample64 >= reader->lengthInSamples) + return makePolyAnalysisError(clipId, "Audio clip has no readable audio in its visible range."); + + const int64_t availableSamples64 = reader->lengthInSamples - startSample64; + const int numSamples = static_cast(std::min(requestedSamples64, availableSamples64)); + if (numSamples <= 0) + return makePolyAnalysisError(clipId, "Audio clip has no readable audio in its visible range."); // Read and mix to mono juce::AudioBuffer buffer(static_cast(reader->numChannels), numSamples); - reader->read(&buffer, 0, numSamples, static_cast(startSample), true, true); + if (! reader->read(&buffer, 0, numSamples, static_cast(startSample64), true, true)) + return makePolyAnalysisError(clipId, "Could not read audio file for MIDI conversion."); juce::AudioBuffer mono(1, numSamples); mono.clear(); @@ -16603,11 +16797,14 @@ juce::var AudioEngine::analyzePolyphonic(const juce::String& trackId, const juce mono.addFrom(0, 0, buffer, ch, 0, numSamples, 1.0f / static_cast(buffer.getNumChannels())); // Run polyphonic analysis - auto result = polyPitchDetector.analyze(mono.getReadPointer(0), numSamples, - reader->sampleRate, clipId); - - // Cache result for reuse in polyphonic editing - polyAnalysisCache[clipId] = result; + PolyPitchDetector::PolyAnalysisResult result; + { + const juce::ScopedLock lock(polyAnalysisLock); + result = polyPitchDetector.analyze(mono.getReadPointer(0), numSamples, + reader->sampleRate, clipId); + // Cache result for reuse in polyphonic editing + polyAnalysisCache[clipId] = result; + } return PolyPitchDetector::resultToJSON(result); } @@ -16633,17 +16830,25 @@ juce::var AudioEngine::applyPolyPitchCorrection(const juce::String& trackId, const juce::var& editedNotesJson) { // Check if we have a cached analysis for this clip - auto cacheIt = polyAnalysisCache.find(clipId); - if (cacheIt == polyAnalysisCache.end()) + PolyPitchDetector::PolyAnalysisResult analysisResult; + { + const juce::ScopedLock lock(polyAnalysisLock); + auto cacheIt = polyAnalysisCache.find(clipId); + if (cacheIt != polyAnalysisCache.end()) + analysisResult = cacheIt->second; + } + + if (analysisResult.clipId.isEmpty()) { - // Need to run analysis first analyzePolyphonic(trackId, clipId); - cacheIt = polyAnalysisCache.find(clipId); - if (cacheIt == polyAnalysisCache.end()) - return PolyResynthesizer::resultToJSON("", false); + const juce::ScopedLock lock(polyAnalysisLock); + auto cacheIt = polyAnalysisCache.find(clipId); + if (cacheIt != polyAnalysisCache.end()) + analysisResult = cacheIt->second; } - const auto& analysisResult = cacheIt->second; + if (analysisResult.clipId.isEmpty()) + return PolyResynthesizer::resultToJSON("", false); // Find the clip's audio file auto clips = playbackEngine.getClipSnapshot(); @@ -16729,16 +16934,25 @@ juce::var AudioEngine::soloPolyNote(const juce::String& trackId, const juce::String& noteId) { // Check cache - auto cacheIt = polyAnalysisCache.find(clipId); - if (cacheIt == polyAnalysisCache.end()) + PolyPitchDetector::PolyAnalysisResult analysisResult; + { + const juce::ScopedLock lock(polyAnalysisLock); + auto cacheIt = polyAnalysisCache.find(clipId); + if (cacheIt != polyAnalysisCache.end()) + analysisResult = cacheIt->second; + } + + if (analysisResult.clipId.isEmpty()) { analyzePolyphonic(trackId, clipId); - cacheIt = polyAnalysisCache.find(clipId); - if (cacheIt == polyAnalysisCache.end()) - return PolyResynthesizer::resultToJSON("", false); + const juce::ScopedLock lock(polyAnalysisLock); + auto cacheIt = polyAnalysisCache.find(clipId); + if (cacheIt != polyAnalysisCache.end()) + analysisResult = cacheIt->second; } - const auto& analysisResult = cacheIt->second; + if (analysisResult.clipId.isEmpty()) + return PolyResynthesizer::resultToJSON("", false); // Find clip audio auto clips = playbackEngine.getClipSnapshot(); @@ -17009,6 +17223,7 @@ void AudioEngine::cancelAiToolsInstall() } juce::var AudioEngine::startAIGeneration(const juce::String& trackId, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJSON) { @@ -17026,51 +17241,71 @@ juce::var AudioEngine::startAIGeneration(const juce::String& trackId, auto aiToolsStatus = stemSeparator.getAiToolsStatus(); if (auto* statusObject = aiToolsStatus.getDynamicObject()) { - const auto musicGenerationReady = static_cast(statusObject->getProperty("musicGenerationReady")); - const auto layoutValid = static_cast(statusObject->getProperty("musicGenerationLayoutValid")); - const auto performanceReady = ! statusObject->hasProperty("musicGenerationPerformanceReady") - || static_cast(statusObject->getProperty("musicGenerationPerformanceReady")); - const auto availableProfilesVar = statusObject->getProperty("musicGenerationAvailableProfiles"); - auto hasNativeMusicProfile = true; - if (auto* availableProfilesArray = availableProfilesVar.getArray()) + if (modelId == "stable-audio-3-medium") + { + auto stableReady = false; + juce::String stableMessage = "Stable Audio 3 Medium is not set up yet."; + if (auto* musicModels = statusObject->getProperty("musicModels").getDynamicObject()) + { + if (auto* stableModel = musicModels->getProperty("stable-audio-3-medium").getDynamicObject()) + { + stableReady = static_cast(stableModel->getProperty("ready")); + stableMessage = stableModel->getProperty("message").toString(); + } + } + + if (! stableReady) + { + result->setProperty("started", false); + result->setProperty("error", stableMessage.isNotEmpty() + ? stableMessage + : "Import Stable Audio 3 Medium from AI Tools Setup before generating."); + return juce::var(result.release()); + } + } + else { - if (! availableProfilesArray->isEmpty()) + const auto musicGenerationReady = static_cast(statusObject->getProperty("musicGenerationReady")); + const auto layoutValid = static_cast(statusObject->getProperty("musicGenerationLayoutValid")); + const auto availableProfilesVar = statusObject->getProperty("musicGenerationAvailableProfiles"); + auto hasNativeMusicProfile = true; + if (auto* availableProfilesArray = availableProfilesVar.getArray()) { - hasNativeMusicProfile = false; - for (const auto& profile : *availableProfilesArray) + if (! availableProfilesArray->isEmpty()) { - if (profile.toString() == "native-xl-turbo") + hasNativeMusicProfile = false; + for (const auto& profile : *availableProfilesArray) { - hasNativeMusicProfile = true; - break; + if (profile.toString() == "ace-diffusers") + { + hasNativeMusicProfile = true; + break; + } } } } - } - if (! musicGenerationReady || ! layoutValid || ! performanceReady || ! hasNativeMusicProfile) - { - const auto musicGenerationStatusMessage = statusObject->getProperty("musicGenerationStatusMessage").toString(); - const auto musicGenerationPerformanceStatusMessage = statusObject->getProperty("musicGenerationPerformanceStatusMessage").toString(); - const auto statusMessage = statusObject->getProperty("message").toString(); - const auto errorMessage = statusObject->getProperty("error").toString(); - const auto modelId = statusObject->getProperty("musicGenerationModelId").toString(); - const auto checkpointRoot = statusObject->getProperty("musicGenerationCheckpointRoot").toString(); - result->setProperty("started", false); - result->setProperty( - "error", - errorMessage.isNotEmpty() - ? errorMessage - : (! hasNativeMusicProfile) - ? "The Native XL Turbo ACE-Step profile is still missing required music-generation assets. Retry AI Tools install to finish setup." - : musicGenerationPerformanceStatusMessage.isNotEmpty() - ? musicGenerationPerformanceStatusMessage - : musicGenerationStatusMessage.isNotEmpty() - ? musicGenerationStatusMessage - : (! layoutValid && modelId.isNotEmpty() && checkpointRoot.isNotEmpty()) - ? "Pinned ACE-Step model " + modelId + " is not ready in " + checkpointRoot + "." - : statusMessage); - return juce::var(result.release()); + if (! musicGenerationReady || ! layoutValid || ! hasNativeMusicProfile) + { + const auto musicGenerationStatusMessage = statusObject->getProperty("musicGenerationStatusMessage").toString(); + const auto statusMessage = statusObject->getProperty("message").toString(); + const auto errorMessage = statusObject->getProperty("error").toString(); + const auto aceModelId = statusObject->getProperty("musicGenerationModelId").toString(); + const auto checkpointRoot = statusObject->getProperty("musicGenerationCheckpointRoot").toString(); + result->setProperty("started", false); + result->setProperty( + "error", + errorMessage.isNotEmpty() + ? errorMessage + : (! hasNativeMusicProfile) + ? "The ACE-Step Diffusers backend is still missing required music-generation assets. Retry AI Tools install to finish setup." + : musicGenerationStatusMessage.isNotEmpty() + ? musicGenerationStatusMessage + : (! layoutValid && aceModelId.isNotEmpty() && checkpointRoot.isNotEmpty()) + ? "ACE-Step Diffusers model " + aceModelId + " is not ready in " + checkpointRoot + "." + : statusMessage); + return juce::var(result.release()); + } } } @@ -17078,7 +17313,7 @@ juce::var AudioEngine::startAIGeneration(const juce::String& trackId, .getChildFile("OpenStudio") .getChildFile("generated-music"); - if (! aiTrackEngine.startGeneration(workflowId, paramsJSON, outputDir)) + if (! aiTrackEngine.startGeneration(modelId, workflowId, paramsJSON, outputDir)) { auto progress = aiTrackEngine.pollProgress(); result->setProperty("started", false); @@ -17102,6 +17337,12 @@ juce::var AudioEngine::getAIGenerationProgress() obj->setProperty("phase", progress.phase); obj->setProperty("message", progress.message); obj->setProperty("backend", progress.backend); + if (progress.modelId.isNotEmpty()) + obj->setProperty("modelId", progress.modelId); + if (progress.workflowId.isNotEmpty()) + obj->setProperty("workflowId", progress.workflowId); + if (progress.sourceClipId.isNotEmpty()) + obj->setProperty("sourceClipId", progress.sourceClipId); obj->setProperty("elapsedMs", progress.elapsedMs); obj->setProperty("heartbeatTs", progress.heartbeatTs); if (progress.phaseProgress >= 0.0) diff --git a/Source/AudioEngine.h b/Source/AudioEngine.h index e3e6e81..6344837 100644 --- a/Source/AudioEngine.h +++ b/Source/AudioEngine.h @@ -274,6 +274,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, // Waveform Visualization juce::var getWaveformPeaks(const juce::String& filePath, int samplesPerPixel, int startSample, int numPixels); + bool refreshWaveformPeaks(const juce::String& filePath); juce::var getRecordingPeaks(const juce::String& trackId, int samplesPerPixel, int numPixels); // Offline Render/Export @@ -524,6 +525,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, void cancelStemSeparation(); void cancelAiToolsInstall(); juce::var startAIGeneration(const juce::String& trackId, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJSON); juce::var getAIGenerationProgress(); @@ -642,7 +644,8 @@ class AudioEngine : public juce::AudioIODeviceCallback, // FFmpeg helpers for lossy encoding and sample rate conversion juce::File findFFmpegExe() const; bool convertWithFFmpeg(const juce::File& inputFile, const juce::File& outputFile, - const juce::String& format, double targetSampleRate, int quality) const; + const juce::String& format, double targetSampleRate, int quality, + int bitDepth, int numChannels) const; // Device settings persistence void saveDeviceSettings(); void loadDeviceSettings(); @@ -872,6 +875,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, // Polyphonic Pitch Detection (Phase 6) — lazy-loaded PolyPitchDetector polyPitchDetector; bool polyModelLoadAttempted = false; + juce::CriticalSection polyAnalysisLock; // Polyphonic Pitch Editing (Phase 7) PolyResynthesizer polyResynthesizer; diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index ee745d6..927b342 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -2191,6 +2191,13 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::Array()); } }) + .withNativeFunction ("refreshWaveformPeaks", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + if (args.size() >= 1) { + completion(audioEngine.refreshWaveformPeaks(args[0].toString())); + } else { + completion(false); + } + }) .withNativeFunction ("getRecordingPeaks", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() == 3) { juce::String trackId = args[0].toString(); @@ -2514,6 +2521,50 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::URL(args[0].toString()).launchInDefaultBrowser()); }) + .withNativeFunction ("browseForFile", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const auto title = args.size() > 0 && args[0].isString() + ? args[0].toString() + : juce::String("Select File"); + const auto filters = args.size() > 1 && args[1].isString() + ? args[1].toString() + : juce::String("*"); + auto initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + + fileChooser = std::make_unique( + title, + initialDir, + filters.isNotEmpty() ? filters : juce::String("*"), + true); + + const auto chooserFlags = juce::FileBrowserComponent::openMode + | juce::FileBrowserComponent::canSelectFiles; + fileChooser->launchAsync(chooserFlags, [completion] (const juce::FileChooser& fc) { + auto result = fc.getResult(); + completion(result.existsAsFile() ? result.getFullPathName() : juce::String()); + }); + }) + .withNativeFunction ("browseForFolder", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const auto title = args.size() > 0 && args[0].isString() + ? args[0].toString() + : juce::String("Select Folder"); + auto initialDir = juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile("Downloads"); + if (! initialDir.isDirectory()) + initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + + fileChooser = std::make_unique( + title, + initialDir, + "*", + true); + + const auto chooserFlags = juce::FileBrowserComponent::openMode + | juce::FileBrowserComponent::canSelectDirectories; + fileChooser->launchAsync(chooserFlags, [completion] (const juce::FileChooser& fc) { + auto result = fc.getResult(); + completion(result.isDirectory() ? result.getFullPathName() : juce::String()); + }); + }) // ========== Project Save/Load (F2) ========== .withNativeFunction ("showSaveDialog", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Show native save file dialog @@ -5976,18 +6027,34 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, { auto trackId = args[0].toString(); auto clipId = args[1].toString(); - // Run on background thread to avoid blocking UI - std::thread([this, trackId, clipId, completion]() { - auto result = audioEngine.analyzePolyphonic(trackId, clipId); - completion(result); - }).detach(); + auto* engine = &audioEngine; + juce::Component::SafePointer safeThis(this); + polyAnalysisBridgePool.addJob([engine, safeThis, trackId, clipId, completion]() mutable { + auto result = engine->analyzePolyphonic(trackId, clipId); + juce::MessageManager::callAsync([safeThis, completion, result]() mutable { + if (safeThis != nullptr) + completion(result); + }); + }); } else completion(juce::var()); }) .withNativeFunction ("extractMidiFromAudio", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { if (args.size() >= 2) - completion(audioEngine.extractMidiFromAudio(args[0].toString(), args[1].toString())); + { + auto trackId = args[0].toString(); + auto clipId = args[1].toString(); + auto* engine = &audioEngine; + juce::Component::SafePointer safeThis(this); + polyAnalysisBridgePool.addJob([engine, safeThis, trackId, clipId, completion]() mutable { + auto result = engine->extractMidiFromAudio(trackId, clipId); + juce::MessageManager::callAsync([safeThis, completion, result]() mutable { + if (safeThis != nullptr) + completion(result); + }); + }); + } else completion(juce::var()); }) @@ -6155,8 +6222,10 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::var()); }) .withNativeFunction ("startAIGeneration", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { - if (args.size() >= 3) - completion(audioEngine.startAIGeneration(args[0].toString(), args[1].toString(), args[2].toString())); + if (args.size() >= 4) + completion(audioEngine.startAIGeneration(args[0].toString(), args[1].toString(), args[2].toString(), args[3].toString())); + else if (args.size() >= 3) + completion(audioEngine.startAIGeneration(args[0].toString(), "ace-step-v15-xl-turbo", args[1].toString(), args[2].toString())); else completion(juce::var()); }) @@ -6696,6 +6765,7 @@ bool MainComponent::completePitchRegressionJob(const juce::var& result) MainComponent::~MainComponent() { stopTimer(); + polyAnalysisBridgePool.removeAllJobs(true, 5000); mediaPreviewPool.removeAllJobs(true, 2000); #if JUCE_WINDOWS externalMediaDropTarget.reset(); diff --git a/Source/MainComponent.h b/Source/MainComponent.h index f5ba40b..9f2c574 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -144,6 +144,7 @@ class MainComponent : public juce::Component, std::atomic pitchAnalysisGeneration { 0 }; std::atomic pitchNoteHqPriorityGeneration { 0 }; juce::ThreadPool pitchAnalysisPool { 1 }; + juce::ThreadPool polyAnalysisBridgePool { 1 }; juce::var lastPitchAnalysisResult; // Cached result for fetch-after-event pattern juce::CriticalSection pitchResultLock; diff --git a/Source/Metronome.cpp b/Source/Metronome.cpp index 91ce408..7080f2a 100644 --- a/Source/Metronome.cpp +++ b/Source/Metronome.cpp @@ -22,6 +22,8 @@ Metronome::~Metronome() void Metronome::prepareToPlay(double newSampleRate, int samplesPerBlock) { + juce::ignoreUnused(samplesPerBlock); + if (sampleRate != newSampleRate) { sampleRate = newSampleRate; @@ -68,7 +70,8 @@ void Metronome::getNextAudioBlock(juce::AudioBuffer& buffer, double curre if (!enabled) return; int numSamples = buffer.getNumSamples(); - double samplesPerBeat = (60.0 / bpm) * sampleRate; + const double denominatorScale = denominator > 0 ? (4.0 / static_cast(denominator)) : 1.0; + double samplesPerBeat = (60.0 / bpm) * sampleRate * denominatorScale; // Safety check if (samplesPerBeat <= 0.0) return; @@ -76,22 +79,13 @@ void Metronome::getNextAudioBlock(juce::AudioBuffer& buffer, double curre auto* leftConfig = buffer.getWritePointer(0); auto* rightConfig = buffer.getNumChannels() > 1 ? buffer.getWritePointer(1) : nullptr; - // Detect playback restart: if position jumped backwards (e.g., from >0 back to 0) - // or if this is the very first call (lastSamplePosition is -1) - bool playbackRestarted = (lastSamplePosition < 0) || (currentSamplePosition < lastSamplePosition); - - // If playback restarted at or near position 0, immediately trigger the first click - if (playbackRestarted && currentSamplePosition < samplesPerBeat && !isClicking) + const bool transportDiscontinuity = lastSamplePosition < 0.0 + || std::abs(currentSamplePosition - lastSamplePosition) > 1.0; + if (transportDiscontinuity) { - // Force the first beat to be triggered - isClicking = true; + isClicking = false; clickSampleCounter = 0; - // First beat is always accented (beat 0) - if (!accentBeats.empty()) { - isHighClick = accentBeats[0]; - } else { - isHighClick = true; - } + isHighClick = false; } for (int i = 0; i < numSamples; ++i) @@ -106,7 +100,7 @@ void Metronome::getNextAudioBlock(juce::AudioBuffer& buffer, double curre double prevBeatPos = (currentPos - 1.0) / samplesPerBeat; int currentBeatIdx = static_cast(std::floor(beatPos)); int prevBeatIdx = static_cast(std::floor(prevBeatPos)); - bool isBeatStart = (currentBeatIdx > prevBeatIdx) && (currentPos > 0 || !playbackRestarted); + bool isBeatStart = currentBeatIdx > prevBeatIdx; if (isBeatStart && !isClicking) { diff --git a/Source/PeakCache.cpp b/Source/PeakCache.cpp index 5ba53d0..1477355 100644 --- a/Source/PeakCache.cpp +++ b/Source/PeakCache.cpp @@ -59,6 +59,29 @@ bool PeakCache::hasCachedPeaks(const juce::File& audioFile) const return true; } +void PeakCache::invalidate(const juce::File& audioFile) +{ + const auto key = audioFile.getFullPathName(); + + { + const juce::ScopedLock sl(cacheLock); + memoryCache.erase(key); + } + + { + const juce::ScopedLock sl(pendingLock); + pendingGenerations.erase(key); + } + + auto peakFile = getPeakFilePath(audioFile); + if (peakFile.existsAsFile()) + peakFile.deleteFile(); + + auto legacyPeakFile = getLegacyPeakFilePath(audioFile); + if (legacyPeakFile.existsAsFile()) + legacyPeakFile.deleteFile(); +} + juce::var PeakCache::getPeaks(const juce::File& audioFile, int samplesPerPixel, int startSample, diff --git a/Source/PeakCache.h b/Source/PeakCache.h index d6b5616..aaf6bcb 100644 --- a/Source/PeakCache.h +++ b/Source/PeakCache.h @@ -42,6 +42,9 @@ class PeakCache /** Check if a valid, up-to-date cache exists for this audio file. */ bool hasCachedPeaks(const juce::File& audioFile) const; + /** Drop in-memory and sidecar peak data for this audio file. */ + void invalidate(const juce::File& audioFile); + /** * Generate peak cache on a background thread. * Calls onComplete (on the message thread) when done. diff --git a/Source/StemSeparator.cpp b/Source/StemSeparator.cpp index 94ba435..79484a1 100644 --- a/Source/StemSeparator.cpp +++ b/Source/StemSeparator.cpp @@ -7,9 +7,11 @@ namespace { constexpr auto kStemModelName = "BS-Roformer-SW.ckpt"; -constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; -constexpr auto kPinnedMusicGenerationModelRepoId = "ACE-Step/acestep-v15-xl-turbo"; -constexpr auto kPinnedMusicGenerationSharedRepoId = "ACE-Step/Ace-Step1.5"; +constexpr auto kPinnedMusicGenerationModelId = "ace-step-v15-xl-turbo"; +constexpr auto kPinnedMusicGenerationModelRepoId = "ACE-Step/acestep-v15-xl-turbo-diffusers"; +constexpr auto kPinnedMusicGenerationSharedRepoId = "ACE-Step/acestep-v15-xl-turbo-diffusers"; +constexpr auto kStableAudioModelId = "stable-audio-3-medium"; +constexpr auto kStableAudioAttribution = "Powered by Stability AI"; constexpr auto kFeatureStemSeparation = "stemSeparation"; constexpr auto kFeatureAudioGeneration = "audioGeneration"; constexpr juce::int64 kMinStemSystemRamMb = 8 * 1024; @@ -67,6 +69,23 @@ juce::StringArray varToStringArray (const juce::var& value) return result; } +juce::StringArray getStableAudioRequiredRelativePaths() +{ + return { + "model.safetensors", + "model_config.json", + "LICENSE.md", + "LICENSE_GEMMA.md", + "NOTICE", + "t5gemma-b-b-ul2/model.safetensors", + "t5gemma-b-b-ul2/config.json", + "t5gemma-b-b-ul2/tokenizer.json", + "t5gemma-b-b-ul2/tokenizer.model", + "t5gemma-b-b-ul2/tokenizer_config.json", + "t5gemma-b-b-ul2/special_tokens_map.json", + }; +} + juce::String normaliseFeatureId (juce::String value) { value = value.trim(); @@ -144,7 +163,7 @@ bool isInstallerTerminalFailureCode (const juce::String& errorCode) bool hasNativeMusicProfile (const StemSeparator::AiToolsStatus& status) { return status.musicGenerationAvailableProfiles.isEmpty() - || status.musicGenerationAvailableProfiles.contains("native-xl-turbo"); + || status.musicGenerationAvailableProfiles.contains("ace-diffusers"); } juce::String sanitiseArchiveEntryName (juce::String name) @@ -262,6 +281,11 @@ juce::File StemSeparator::getUserRuntimeRoot() const return getUserDataRoot().getChildFile("stem-runtime"); } +juce::File StemSeparator::getStableAudioRuntimeRoot() const +{ + return getUserDataRoot().getChildFile("stable-audio-runtime"); +} + juce::File StemSeparator::getUserModelsDir() const { return getUserDataRoot().getChildFile("models"); @@ -272,7 +296,17 @@ juce::File StemSeparator::getMusicGenerationCheckpointRoot() const return juce::File::getSpecialLocation(juce::File::userHomeDirectory) .getChildFile(".cache") .getChildFile("ace-step") - .getChildFile("checkpoints"); + .getChildFile("diffusers"); +} + +juce::File StemSeparator::getStableAudioModelRoot() const +{ + return getUserModelsDir().getChildFile(kStableAudioModelId); +} + +juce::File StemSeparator::findStableAudioPython() const +{ + return findPythonInRuntimeRoot(getStableAudioRuntimeRoot()); } juce::File StemSeparator::getAiToolsInstallLogFile() const @@ -694,6 +728,28 @@ bool StemSeparator::hasRequiredModel(const juce::File& modelsDir) const return modelsDir.isDirectory() && modelsDir.getChildFile(kStemModelName).existsAsFile(); } +juce::StringArray StemSeparator::getMissingStableAudioFiles (const juce::File& modelRoot) const +{ + juce::StringArray missing; + if (! modelRoot.isDirectory()) + { + missing.add(modelRoot.getFullPathName()); + return missing; + } + + for (const auto& relativePath : getStableAudioRequiredRelativePaths()) + { + if (! modelRoot.getChildFile(relativePath).existsAsFile()) + missing.add(relativePath); + } + return missing; +} + +bool StemSeparator::isStableAudioModelFolderValid (const juce::File& modelRoot) const +{ + return getMissingStableAudioFiles(modelRoot).isEmpty(); +} + bool StemSeparator::isExternalPythonFallbackEnabled() const { #if OPENSTUDIO_ENABLE_EXTERNAL_PYTHON_AI_FALLBACK @@ -703,6 +759,15 @@ bool StemSeparator::isExternalPythonFallbackEnabled() const #endif } +bool StemSeparator::hasStableAudioRuntime() const +{ + const auto python = findStableAudioPython(); + return python.existsAsFile() + && getStableAudioRuntimeRoot() + .getChildFile(".openstudio-stable-audio-ready") + .existsAsFile(); +} + StemSeparator::InstallOptions StemSeparator::parseInstallOptions (const juce::String& optionsJson, bool legacyConfirmed) const { InstallOptions options; @@ -723,9 +788,18 @@ StemSeparator::InstallOptions StemSeparator::parseInstallOptions (const juce::St if (parsed.hasProperty("requestedFeature")) options.requestedFeature = normaliseFeatureId(parsed["requestedFeature"].toString()); + if (parsed.hasProperty("modelId")) + options.modelId = parsed["modelId"].toString().trim(); + if (parsed.hasProperty("selectedFeatures")) options.selectedFeatures = normaliseFeatureArray(parsed["selectedFeatures"]); + if (parsed.hasProperty("stableAudioModelPath")) + options.stableAudioModelPath = parsed["stableAudioModelPath"].toString(); + + if (parsed.hasProperty("stableAudioLicenseAccepted")) + options.stableAudioLicenseAccepted = static_cast(parsed["stableAudioLicenseAccepted"]); + if (options.selectedFeatures.isEmpty() && options.requestedFeature.isNotEmpty()) options.selectedFeatures.add(options.requestedFeature); @@ -897,7 +971,6 @@ juce::var StemSeparator::buildFeatureStatusVar (const AiToolsStatus& status, con const auto stemReady = status.scriptAvailable && status.runtimeInstalled && status.modelInstalled; const auto audioReady = status.musicGenerationReady && status.musicGenerationLayoutValid - && status.musicGenerationPerformanceReady && hasNativeMusicProfile(status); auto features = std::make_unique(); @@ -961,7 +1034,6 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File status.progress = 1.0f; const auto musicGenerationFullyReady = status.musicGenerationReady && status.musicGenerationLayoutValid - && status.musicGenerationPerformanceReady && hasNativeMusicProfile(status); status.stepLabel = musicGenerationFullyReady ? "AI tools are ready." : "Stem separation is ready."; status.stepIndex = 0; @@ -979,16 +1051,24 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File status.message = musicGenerationFullyReady ? "AI tools are ready." : (! hasNativeMusicProfile(status) - ? "Stem separation is ready, but the Native XL Turbo ACE-Step profile is still missing required music-generation assets." + ? "Stem separation is ready, but the ACE-Step Diffusers backend is still missing required music-generation assets." : (status.musicGenerationPerformanceStatusMessage.isNotEmpty() ? status.musicGenerationPerformanceStatusMessage : (status.musicGenerationStatusMessage.isNotEmpty() ? status.musicGenerationStatusMessage - : "Stem separation is ready, but music generation still needs a compatible ACE-Step runtime bridge."))); + : "Stem separation is ready, but music generation still needs the ACE-Step Diffusers runtime."))); status.activityLines.clear(); status.activityLines.add(status.stepLabel); if (status.message != status.stepLabel) status.activityLines.add(status.message); + if (musicGenerationFullyReady + && ! status.musicGenerationPerformanceReady + && status.musicGenerationPerformanceStatusMessage.isNotEmpty()) + { + status.statusWarning = status.musicGenerationPerformanceStatusMessage; + status.statusWarningCode = "music_generation_acceleration_warning"; + status.activityLines.add(status.statusWarning); + } if (status.installedFeatures.isEmpty()) status.installedFeatures.add(kFeatureStemSeparation); if (musicGenerationFullyReady) @@ -1280,7 +1360,6 @@ void StemSeparator::scheduleStatusRefresh() refreshedStatus.installedFeatures.add(kFeatureStemSeparation); if (refreshedStatus.musicGenerationReady && refreshedStatus.musicGenerationLayoutValid - && refreshedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(refreshedStatus)) refreshedStatus.installedFeatures.add(kFeatureAudioGeneration); const auto refreshedHardware = probeHardwareStatus(); @@ -1288,7 +1367,6 @@ void StemSeparator::scheduleStatusRefresh() refreshedStatus.features = buildFeatureStatusVar(refreshedStatus, refreshedHardware); const auto musicGenerationFullyReady = refreshedStatus.musicGenerationReady && refreshedStatus.musicGenerationLayoutValid - && refreshedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(refreshedStatus); const auto isReconciliationRefresh = previousStatus.statusWarningCode == "reconciling_install_state"; @@ -1626,7 +1704,7 @@ bool StemSeparator::extractRuntimeArchive (const juce::File& archiveFile, return true; } -juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) +juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) const { auto obj = std::make_unique(); juce::Array supportedBackends; @@ -1709,6 +1787,55 @@ juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) obj->setProperty("requestedFeature", status.requestedFeature); obj->setProperty("hardware", status.hardware); obj->setProperty("features", status.features); + + auto musicModels = std::make_unique(); + const auto aceReady = status.musicGenerationReady + && status.musicGenerationLayoutValid + && hasNativeMusicProfile(status); + auto aceModel = std::make_unique(); + aceModel->setProperty("id", kPinnedMusicGenerationModelId); + aceModel->setProperty("label", "ACE-Step 1.5 XL Turbo"); + aceModel->setProperty("installed", status.musicGenerationReady && status.musicGenerationLayoutValid); + aceModel->setProperty("ready", aceReady); + aceModel->setProperty("compatible", true); + aceModel->setProperty("blocked", ! aceReady); + aceModel->setProperty("blockReason", aceReady ? juce::String() : status.musicGenerationStatusMessage); + aceModel->setProperty("message", aceReady ? "ACE-Step is ready." : status.musicGenerationStatusMessage); + aceModel->setProperty("modelPath", status.musicGenerationCheckpointRoot); + musicModels->setProperty(kPinnedMusicGenerationModelId, juce::var(aceModel.release())); + + const auto stableRoot = getStableAudioModelRoot(); + const auto stableMissing = getMissingStableAudioFiles(stableRoot); + const auto stableModelReady = stableMissing.isEmpty(); + const auto stableRuntimeReady = hasStableAudioRuntime(); + const auto stableReady = stableModelReady && stableRuntimeReady; + auto stableModel = std::make_unique(); + stableModel->setProperty("id", kStableAudioModelId); + stableModel->setProperty("label", "Stable Audio 3 Medium"); + stableModel->setProperty("installed", stableReady); + stableModel->setProperty("ready", stableReady); + stableModel->setProperty("compatible", true); + stableModel->setProperty("blocked", ! stableReady); + stableModel->setProperty("blockReason", + stableReady ? juce::String() + : stableModelReady ? "Stable Audio 3 runtime dependencies are not installed." + : "Stable Audio 3 Medium has not been imported."); + stableModel->setProperty("message", + stableReady ? "Stable Audio 3 Medium is ready." + : stableModelReady ? "Install the separate Stable Audio 3 runtime dependencies." + : "Import the Stable Audio 3 Medium Hugging Face snapshot."); + stableModel->setProperty("modelPath", stableRoot.getFullPathName()); + stableModel->setProperty("runtimePath", getStableAudioRuntimeRoot().getFullPathName()); + stableModel->setProperty("runtimeReady", stableRuntimeReady); + stableModel->setProperty("modelReady", stableModelReady); + stableModel->setProperty("attribution", kStableAudioAttribution); + stableModel->setProperty("licenseAccepted", stableModelReady); + juce::Array stableMissingArray; + for (const auto& missing : stableMissing) + stableMissingArray.add(missing); + stableModel->setProperty("missingFiles", stableMissingArray); + musicModels->setProperty(kStableAudioModelId, juce::var(stableModel.release())); + obj->setProperty("musicModels", juce::var(musicModels.release())); return juce::var(obj.release()); } @@ -1753,10 +1880,600 @@ juce::var StemSeparator::installAiTools (const juce::String& optionsJson) auto cachedStatus = getCachedAiToolsStatusSnapshot(); const bool musicGenerationFullyReady = cachedStatus.musicGenerationReady && cachedStatus.musicGenerationLayoutValid - && cachedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(cachedStatus); auto result = std::make_unique(); + + const bool stableAudioSetupRequested = installOptions.modelId == kStableAudioModelId + || installOptions.stableAudioModelPath.isNotEmpty(); + + if (stableAudioSetupRequested) + { + const auto stableSessionId = juce::Uuid().toString(); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_requested", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("modelId", kStableAudioModelId); + payload.setProperty("sourcePath", installOptions.stableAudioModelPath); + })); + + if (cachedStatus.installInProgress) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "install_already_running"); + })); + result->setProperty("started", false); + result->setProperty("error", "AI tools installation is already running."); + result->setProperty("status", aiToolsStatusToVar(cachedStatus)); + return juce::var(result.release()); + } + + if (installOptions.stableAudioModelPath.isEmpty()) + { + auto status = cachedStatus; + status.state = "error"; + status.progress = 0.0f; + status.installInProgress = false; + status.message = "Choose the downloaded Stable Audio 3 Medium snapshot folder before setup."; + status.error = status.message; + status.errorCode = "stable_audio_model_path_required"; + status.lastPhase = "stable_audio_import"; + status.terminalReason = status.errorCode; + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + { + const juce::ScopedLock lock(aiToolsStatusLock); + lastAiToolsStatus = status; + initialStatusPrepared = true; + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "missing_source_path"); + })); + result->setProperty("started", false); + result->setProperty("error", status.error); + result->setProperty("status", aiToolsStatusToVar(status)); + return juce::var(result.release()); + } + + if (! installOptions.stableAudioLicenseAccepted) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "license_not_accepted"); + })); + result->setProperty("started", false); + result->setProperty("error", "Stable Audio 3 setup requires accepting the Stability AI and Gemma license notices before import."); + result->setProperty("status", aiToolsStatusToVar(cachedStatus)); + return juce::var(result.release()); + } + + const juce::File sourceRoot(installOptions.stableAudioModelPath); + const auto missingFiles = getMissingStableAudioFiles(sourceRoot); + if (! missingFiles.isEmpty()) + { + auto status = cachedStatus; + status.state = "error"; + status.progress = 0.0f; + status.installInProgress = false; + status.message = "The selected Stable Audio 3 folder is missing required files."; + status.error = "Missing files: " + missingFiles.joinIntoString(", "); + status.errorCode = "stable_audio_model_layout_invalid"; + status.lastPhase = "stable_audio_import"; + status.terminalReason = status.errorCode; + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + { + const juce::ScopedLock lock(aiToolsStatusLock); + lastAiToolsStatus = status; + initialStatusPrepared = true; + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_source_layout_invalid", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", sourceRoot.getFullPathName()); + juce::Array missing; + for (const auto& file : missingFiles) + missing.add(file); + payload.setProperty("missingFiles", missing); + })); + result->setProperty("started", false); + result->setProperty("error", status.error); + result->setProperty("status", aiToolsStatusToVar(status)); + return juce::var(result.release()); + } + + aiToolsCancelRequested = false; + aiToolsInstallWorkInProgress = true; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = 0.05f; + status.installInProgress = true; + status.message = "Importing Stable Audio 3 Medium snapshot..."; + status.stepLabel = "Importing Stable Audio 3 Medium"; + status.lastPhase = "stable_audio_import"; + status.error.clear(); + status.errorCode.clear(); + status.activityLines.clear(); + status.activityLines.add("Valid Stable Audio 3 source: " + sourceRoot.getFullPathName()); + status.activityLines.add("Copying snapshot into the managed OpenStudio model folder."); + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + status.installSessionId = stableSessionId; + status.selectedFeatures.clear(); + status.selectedFeatures.add(kFeatureAudioGeneration); + status.requestedFeatures = status.selectedFeatures; + status.requestedFeature = kFeatureAudioGeneration; + }); + + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", sourceRoot.getFullPathName()); + payload.setProperty("destinationPath", getStableAudioModelRoot().getFullPathName()); + payload.setProperty("runtimePath", getStableAudioRuntimeRoot().getFullPathName()); + })); + + std::thread([this, sourcePath = sourceRoot.getFullPathName(), stableSessionId]() + { + const juce::File source(sourcePath); + const auto destination = getStableAudioModelRoot(); + const auto runtimeRoot = getStableAudioRuntimeRoot(); + const auto readyMarker = runtimeRoot.getChildFile(".openstudio-stable-audio-ready"); + juce::String error; + bool success = true; + auto progressForStableAudioCommand = [] (const juce::String& label) + { + if (label.containsIgnoreCase("Creating separate")) + return 0.25f; + if (label.containsIgnoreCase("Updating Stable Audio")) + return 0.35f; + if (label.containsIgnoreCase("runtime dependencies")) + return 0.50f; + if (label.containsIgnoreCase("Flash Attention")) + return 0.78f; + if (label.containsIgnoreCase("Validating")) + return 0.92f; + return 0.40f; + }; + auto runCommand = [&] (const juce::StringArray& command, + const juce::String& label, + int timeoutMs) -> bool + { + const auto commandStartedAt = juce::Time::getMillisecondCounterHiRes(); + const auto commandProgress = progressForStableAudioCommand(label); + auto lastUiUpdateMs = commandStartedAt; + juce::String commandOutput; + auto addOutputToStatus = [&] (const juce::String& outputChunk) + { + if (outputChunk.isEmpty()) + return; + + commandOutput += outputChunk; + juce::StringArray lines; + lines.addLines(outputChunk.replace("\r", "\n")); + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.progress = commandProgress; + status.elapsedMs = static_cast(juce::Time::getMillisecondCounterHiRes() - commandStartedAt); + for (const auto& rawLine : lines) + { + const auto trimmed = rawLine.trim(); + if (trimmed.isNotEmpty()) + status.activityLines.add(trimmed.substring(0, 900)); + } + }); + }; + + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("command", command.joinIntoString(" ")); + })); + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = commandProgress; + status.installInProgress = true; + status.message = label; + status.stepLabel = label; + status.lastPhase = "stable_audio_runtime"; + status.installSessionId = stableSessionId; + status.elapsedMs = 0; + const auto isLargeStableAudioStep = label.containsIgnoreCase("runtime dependencies") + || label.containsIgnoreCase("Flash Attention") + || label.containsIgnoreCase("PyTorch"); + status.downloadHint = isLargeStableAudioStep + ? "Stable Audio 3 dependencies can take several minutes to install. OpenStudio will keep updating this log while pip is running." + : juce::String(); + status.isLargeDownload = isLargeStableAudioStep; + status.activityLines.add(label); + }); + + auto process = std::make_shared(); + auto clearStableCommandProcess = [&] + { + const juce::ScopedLock lock(aiToolsStatusLock); + if (installProcess == process) + installProcess.reset(); + }; + { + const juce::ScopedLock lock(aiToolsStatusLock); + installProcess = process; + installCommandLine = command.joinIntoString(" "); + installSessionId = stableSessionId; + installLastObservedPhase = "stable_audio_runtime"; + } + + if (! process->start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) + { + clearStableCommandProcess(); + error = label + " could not be started."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("error", error); + })); + return false; + } + + for (;;) + { + addOutputToStatus(process->readAllProcessOutput()); + + if (! process->isRunning()) + break; + + if (aiToolsCancelRequested.load()) + { + process->kill(); + error = "Stable Audio 3 import was cancelled."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_cancelled", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + })); + clearStableCommandProcess(); + return false; + } + + const auto nowMs = juce::Time::getMillisecondCounterHiRes(); + if (nowMs - commandStartedAt >= static_cast(timeoutMs)) + { + process->kill(); + error = label + " timed out."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("error", error); + })); + clearStableCommandProcess(); + return false; + } + + if (nowMs - lastUiUpdateMs >= 2000.0) + { + lastUiUpdateMs = nowMs; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = commandProgress; + status.installInProgress = true; + status.message = label; + status.stepLabel = label; + status.lastPhase = "stable_audio_runtime"; + status.elapsedMs = static_cast(nowMs - commandStartedAt); + status.activityLines.add(label + " still running..."); + }); + } + + juce::Thread::sleep(250); + } + + addOutputToStatus(process->readAllProcessOutput()); + if (aiToolsCancelRequested.load()) + { + clearStableCommandProcess(); + error = "Stable Audio 3 import was cancelled."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_cancelled", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + })); + return false; + } + const auto output = commandOutput.trim(); + const auto exitCode = process->getExitCode(); + clearStableCommandProcess(); + + if (exitCode != 0) + { + error = label + " failed."; + if (output.isNotEmpty()) + error += " " + output.substring(0, 900); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("exitCode", static_cast(exitCode)); + payload.setProperty("output", output.substring(0, 900)); + })); + return false; + } + + if (output.isNotEmpty()) + { + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.activityLines.add(output.substring(0, 900)); + }); + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_succeeded", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("output", output.substring(0, 900)); + })); + return true; + }; + + if (aiToolsCancelRequested.load()) + { + success = false; + error = "Stable Audio 3 import was cancelled."; + } + else if (source.getFullPathName() != destination.getFullPathName()) + { + if (! destination.getParentDirectory().createDirectory()) + { + success = false; + error = "Could not create managed Stable Audio model directory."; + } + else + { + if (destination.exists()) + destination.deleteRecursively(); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_copy_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + })); + success = source.copyDirectoryTo(destination); + if (! success) + error = "Could not copy Stable Audio 3 files into the managed OpenStudio model folder."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_copy_succeeded" : "stable_audio_copy_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + if (error.isNotEmpty()) + payload.setProperty("error", error); + })); + } + } + + if (success) + { + const auto missingAfterCopy = getMissingStableAudioFiles(destination); + if (! missingAfterCopy.isEmpty()) + { + success = false; + error = "Imported Stable Audio 3 snapshot is missing files: " + missingAfterCopy.joinIntoString(", "); + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_imported_layout_valid" : "stable_audio_imported_layout_invalid", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("destinationPath", destination.getFullPathName()); + juce::Array missing; + for (const auto& file : missingAfterCopy) + missing.add(file); + payload.setProperty("missingFiles", missing); + })); + } + + if (success && ! readyMarker.existsAsFile()) + { + if (! runtimeRoot.createDirectory()) + { + success = false; + error = "Could not create the Stable Audio 3 runtime directory."; + } + + auto runtimePython = findStableAudioPython(); + if (success && ! runtimePython.existsAsFile()) + { + auto bootstrapPython = findSystemPython(); + if (! bootstrapPython.existsAsFile()) + bootstrapPython = findPython(); + + if (! bootstrapPython.existsAsFile()) + { + success = false; + error = "Stable Audio 3 runtime setup needs Python, but no suitable Python executable was found."; + } + else + { + juce::StringArray command; + command.add(bootstrapPython.getFullPathName()); + command.add("-m"); + command.add("venv"); + command.add(runtimeRoot.getFullPathName()); + success = runCommand(command, "Creating separate Stable Audio 3 runtime...", 10 * 60 * 1000); + runtimePython = findStableAudioPython(); + if (success && ! runtimePython.existsAsFile()) + { + success = false; + error = "Stable Audio 3 runtime Python was not created."; + } + } + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("pip"); + success = runCommand(command, "Updating Stable Audio 3 runtime package installer...", 15 * 60 * 1000); + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("--force-reinstall"); + command.add("--index-url"); + command.add("https://download.pytorch.org/whl/cu128"); + command.add("torch==2.7.1"); + command.add("torchaudio==2.7.1"); + success = runCommand(command, "Installing Stable Audio 3 CUDA PyTorch runtime...", 60 * 60 * 1000); + if (! success) + error = "Could not install the Stable Audio 3 CUDA PyTorch runtime. " + error; + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("git+https://github.com/Stability-AI/stable-audio-3.git"); + command.add("soundfile"); + success = runCommand(command, "Installing Stable Audio 3 runtime dependencies...", 60 * 60 * 1000); + } + + if (success) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_flash_attention_skipped", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "flash-attn is optional for Stable Audio 3 on Windows; using PyTorch attention fallback"); + })); + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-c"); + command.add("import stable_audio_3, torch, torchaudio, soundfile; assert torch.cuda.is_available(), 'CUDA PyTorch is not available in the Stable Audio 3 runtime'; print('Stable Audio 3 runtime import check passed'); print('torch=' + torch.__version__ + ', cuda=' + str(torch.version.cuda))"); + success = runCommand(command, "Validating Stable Audio 3 runtime imports...", 5 * 60 * 1000); + } + + if (success && ! readyMarker.replaceWithText("ready\n", false, false, "\n")) + { + success = false; + error = "Could not write Stable Audio 3 runtime readiness marker."; + } + } + + aiToolsInstallWorkInProgress = false; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + const bool cancelled = aiToolsCancelRequested.load() + || error.containsIgnoreCase("cancelled"); + status.state = success ? "ready" : cancelled ? "cancelled" : "error"; + status.progress = success ? 1.0f : 0.0f; + status.installInProgress = false; + status.message = success + ? "Stable Audio 3 Medium is ready." + : error; + status.stepLabel = status.message; + status.lastPhase = "stable_audio_import"; + status.error = success ? juce::String() : error; + status.errorCode = success || cancelled ? juce::String() : "stable_audio_import_failed"; + status.terminalReason = status.errorCode; + status.activityLines.clear(); + status.activityLines.add(status.message); + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + }); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_import_finished" : "stable_audio_import_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("modelId", kStableAudioModelId); + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + payload.setProperty("runtimePath", runtimeRoot.getFullPathName()); + if (error.isNotEmpty()) + payload.setProperty("error", error); + })); + }).detach(); + + result->setProperty("started", true); + result->setProperty("message", "Stable Audio 3 import started."); + result->setProperty("status", aiToolsStatusToVar(getCachedAiToolsStatusSnapshot())); + return juce::var(result.release()); + } + bool selectedFeaturesReady = true; for (const auto& feature : installOptions.selectedFeatures) { diff --git a/Source/StemSeparator.h b/Source/StemSeparator.h index 58fad0c..9a9d761 100644 --- a/Source/StemSeparator.h +++ b/Source/StemSeparator.h @@ -162,21 +162,33 @@ class StemSeparator /** Get the preferred stem-runtime root directory under user app-data. */ juce::File getUserRuntimeRoot() const; + /** Get the separate Stable Audio runtime root directory under user app-data. */ + juce::File getStableAudioRuntimeRoot() const; + /** Get the preferred models directory under user app-data. */ juce::File getUserModelsDir() const; /** Get the pinned ACE-Step checkpoint root used for music generation. */ juce::File getMusicGenerationCheckpointRoot() const; + /** Get the managed Stable Audio 3 Medium snapshot root. */ + juce::File getStableAudioModelRoot() const; + /** Find the prepared user-runtime Python executable. */ juce::File findPython() const; + /** Find the prepared Stable Audio runtime Python executable. */ + juce::File findStableAudioPython() const; + /** Find a user-managed Python interpreter suitable for bootstrapping installs. */ juce::File findSystemPython() const; /** Return true when this build is allowed to use external Python fallback. */ bool isExternalPythonFallbackEnabled() const; + /** Return true when the separate Stable Audio runtime is present. */ + bool hasStableAudioRuntime() const; + /** Resolve a Python executable from a runtime root. */ juce::File findPythonInRuntimeRoot (const juce::File& runtimeRoot) const; @@ -243,6 +255,9 @@ class StemSeparator bool userConfirmedDownload = false; juce::StringArray selectedFeatures; juce::String requestedFeature; + juce::String modelId; + juce::String stableAudioModelPath; + bool stableAudioLicenseAccepted = false; }; struct HardwareStatus @@ -280,6 +295,12 @@ class StemSeparator /** Return true if the preferred stem model is available. */ bool hasRequiredModel (const juce::File& modelsDir) const; + /** Return missing required files for a Stable Audio 3 Medium snapshot. */ + juce::StringArray getMissingStableAudioFiles (const juce::File& modelRoot) const; + + /** Return true if a Stable Audio 3 Medium snapshot contains the required files. */ + bool isStableAudioModelFolderValid (const juce::File& modelRoot) const; + /** Poll a running AI tools install and refresh the cached status. */ void pollInstallProgress(); @@ -333,7 +354,7 @@ class StemSeparator juce::String& errorCode) const; /** Serialize AI tools status to juce::var for the native bridge. */ - static juce::var aiToolsStatusToVar (const AiToolsStatus& status); + juce::var aiToolsStatusToVar (const AiToolsStatus& status) const; std::unique_ptr childProcess; std::shared_ptr installProcess; diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md index 3a18351..c084149 100644 --- a/docs/USER_MANUAL.md +++ b/docs/USER_MANUAL.md @@ -23,6 +23,7 @@ Version 3.0 -- Comprehensive Reference Guide 15. [Customization](#15-customization) 16. [Keyboard Shortcuts](#16-keyboard-shortcuts) 17. [Troubleshooting](#17-troubleshooting) +18. [AI Music and Assisted Audio](#18-ai-music-and-assisted-audio) --- @@ -1062,16 +1063,16 @@ Each built-in OpenStudio effect includes: - Preset management (save/load presets) - Bypass toggle -### 9.4 VST3 Plugin Support +### 9.4 Plugin Hosting -OpenStudio hosts third-party VST3 plugins for both effects and virtual instruments: +OpenStudio hosts third-party plugins for effects and virtual instruments. VST3 is the most mature path. CLAP and LV2 code paths are present and exposed in current builds where the plugin format is available, but individual plugin compatibility may vary more than VST3. **Scanning for plugins:** 1. Go to the FX Chain Panel or Plugin Browser. -2. Click **Scan** to scan standard VST3 directories for installed plugins. +2. Click **Scan** to scan standard plugin directories for installed plugins. 3. Scanned plugins appear in the plugin list organized by manufacturer and category. -**Adding a VST3 plugin:** +**Adding a plugin:** 1. Open the FX Chain Panel for a track. 2. Click the **+** button or "Add Plugin". 3. Browse the plugin list (filterable by name, manufacturer, category). @@ -1079,7 +1080,7 @@ OpenStudio hosts third-party VST3 plugins for both effects and virtual instrumen 5. The plugin's native editor window opens automatically. **Plugin editor windows:** -- VST3 editors open in separate native windows. +- Native plugin editors open in separate native windows when the hosted plugin exposes an editor. - Parameters can be adjusted in the native editor or via the FX Chain Panel's parameter list. ### 9.5 Plugin Presets @@ -1298,10 +1299,9 @@ Choose the time range to render: | **FLAC** | Lossless compressed audio | 16-bit, 24-bit | | **MP3** | Lossy compressed (128-320 kbps) | N/A (bitrate-based) | | **OGG Vorbis** | Lossy compressed (quality 3-10) | N/A (quality-based) | -| **RAW PCM** | Headerless PCM data | 16-bit, 24-bit, 32-bit float | **Sample rate**: 44100, 48000, 88200, 96000, or 192000 Hz. -**Note**: The actual render always processes at the current device sample rate. +**Note**: Rendering processes through the current engine/device configuration, then post-processes when a target sample-rate conversion is requested. **Channels**: Stereo or Mono. @@ -1311,7 +1311,7 @@ Choose the time range to render: |-----------------|------------------------------------------------------------------| | **Normalize** | Peak-normalizes the output to 0 dBFS | | **Dither** | Applies dither when reducing bit depth. Types: TPDF, Noise Shaped. Only available for 16-bit and 24-bit output. | -| **Resample Quality** | Fast, Good, or Best quality for sample rate conversion | +| **Resample Quality** | UI placeholder only in the current build; backend support is pending | ### 12.7 Secondary Output @@ -1323,7 +1323,7 @@ Enable **Secondary output** to simultaneously render a second format (e.g., rend ### 12.8 Metadata -Expand the **Metadata** section to embed information in the rendered file: +The **Metadata** section is present as a disabled placeholder in the current build. Metadata embedding is planned, but these fields are not written yet: - Title - Artist @@ -1337,7 +1337,7 @@ Expand the **Metadata** section to embed information in the rendered file: | Option | Description | |-------------------------------|-----------------------------------------------------| -| **Online render (1x speed)** | Render in real-time instead of offline (for live capture or plugin compatibility) | +| **Online render (1x speed)** | UI placeholder only in the current build; currently disabled | | **Add to project after render** | Automatically import rendered files as new clips in the project | ### 12.10 Render Queue @@ -1470,11 +1470,10 @@ Convert multiple audio files between formats: ### 13.15 Capture Output -Record OpenStudio's master output in real-time: +Live capture is an experimental plumbing path in the current build. The menu action exists, but production-ready real-time master-output capture is not part of the stable user workflow yet: - Go to **File > Capture Output** to toggle live capture. -- Audio is recorded to a file as it plays. -- Useful for capturing improvisations or live performances. +- Treat this as a development/diagnostic feature until the backend capture path is fully enabled. --- @@ -1738,7 +1737,7 @@ Press `Ctrl+Shift+P` to open the Command Palette. Type to fuzzy-search through a If you have 32-bit VST plugins that need to run in the 64-bit OpenStudio environment: - Toggle via **Options > Toggle 32-bit Plugin Bridge**. -- This enables a bridging mechanism to load 32-bit plugins. +- This control is currently experimental. OpenStudio's stable plugin hosting path is 64-bit native plugin hosting. --- @@ -1931,7 +1930,7 @@ If you have 32-bit VST plugins that need to run in the 64-bit OpenStudio environ **Solutions**: 1. Ensure the plugin is installed in a standard VST3 directory. 2. Open the FX Chain Panel and click **Scan** to rescan for plugins. -3. Verify the plugin is a 64-bit VST3 (OpenStudio only supports 64-bit plugins natively; use the 32-bit bridge for older plugins). +3. Verify the plugin is a supported 64-bit plugin. VST3 is the most mature path; CLAP/LV2 compatibility may vary by plugin and build. 4. Check that the plugin file is not corrupted. ### 17.5 Plugin Causing Crashes or Noise @@ -2031,17 +2030,68 @@ If you have 32-bit VST plugins that need to run in the 64-bit OpenStudio environ --- +## 18. AI Music and Assisted Audio + +OpenStudio includes optional AI-assisted workflows that live inside the normal DAW session. These features are not bundled into the base app by default; install the required AI Tools runtime from inside the app when prompted. + +### 18.1 AI Tools Setup + +Use **AI Tools Setup** when a generation or stem workflow reports that its runtime is missing. + +- The installer prepares optional local runtime assets instead of making the base DAW download huge. +- Installation can be cancelled, reset, or retried from the setup modal. +- Generated audio is imported back into the project as normal clips/tracks. + +### 18.2 AI Tracks + +AI tracks are used for prompt-driven generation workflows: + +| Workflow | Description | +|----------|-------------| +| **Text to Music** | Generates a fresh music clip using ACE-Step from style/arrangement prompt, optional lyrics, BPM, duration, time signature, language, key/scale, seed, and generation controls. | +| **Lyrics + Style** | Generates a song guided by both structured lyrics and a musical prompt. | +| **Text to Audio** | Generates audio from a prompt using Stable Audio 3 Medium when that runtime/model is installed. | + +Create an AI track from the Insert menu, the command palette, or the `Ctrl+Alt+T` shortcut if it is still bound to its default. + +### 18.3 Clip AI Workflows + +Right-click an audio clip and open **AI Generation** for source-conditioned workflows: + +| Workflow | Description | +|----------|-------------| +| **Create Variation** | Generates a related version of the selected clip while preserving source identity according to the source/variation controls. | +| **Inpaint Selection** | Regenerates the time selection that overlaps the clip while matching the surrounding audio. Create a time selection first. | +| **Continue Clip** | Generates a continuation tail from the selected clip using the prompt and tail-length/source controls. | + +### 18.4 Stem Separation + +Stem separation splits a source clip into component tracks for remixing, cleanup, practice, or arrangement work: + +- Vocals +- Drums +- Bass +- Other + +The resulting stems are imported back into the session as editable clips. + +### 18.5 Audio to MIDI + +The audio-to-MIDI workflow uses Basic Pitch / ONNX plumbing where available to extract MIDI note data from audio. Use it when you want to turn a recorded or imported performance into MIDI material for editing, layering, or replacement. + +--- + ## Appendix A: Project File Location OpenStudio project files (`.osproj`) are saved to the location you choose when saving. Legacy `.s13` files are still supported. Recorded audio files are stored in a subdirectory alongside the project file. ## Appendix B: Audio Format Support -**Import formats**: WAV, AIFF, FLAC, MP3, OGG Vorbis +**Import formats**: WAV, AIFF, FLAC, MP3, OGG Vorbis, MIDI, and video audio extraction where FFmpeg is available -**Export formats**: WAV, AIFF, FLAC, MP3, OGG Vorbis, RAW PCM +**Export formats**: WAV, AIFF, FLAC, MP3, OGG Vorbis, MIDI, and DDP export -**Plugin format**: VST3 (64-bit native, with optional 32-bit bridge) +**Plugin formats**: VST3 is the stable primary path. CLAP and LV2 code paths are present where available. Experimental 32-bit bridging controls are not part of the stable plugin-hosting path. ## Appendix C: Supported Audio Interfaces diff --git a/docs/features.md b/docs/features.md index 433b85a..fae8fbc 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1,1186 +1,347 @@ -# Studio13-v3 Feature Tracker - -> **Last Updated:** 2026-02-19 (Phase 7E complete: +17 features, 108/160 total) -> **Project Extension:** `.s13` -> **Status Legend:** `[x]` Implemented | `[~]` Partially Implemented | `[ ]` Not Started - ---- - -## Feature Comparison: REAPER vs Studio13 - -This document maps every feature from REAPER's interface (toolbar, menus, render modal) to Studio13's current implementation status. Features marked as remaining include implementation guidance. - ---- - -## I. Top Toolbar - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| New Project | [x] | Ctrl+N, clears state with confirmation | -| Open Project | [x] | Ctrl+O, native file dialog, loads .s13 | -| Save Project | [x] | Ctrl+S, .s13 JSON format with plugin states | -| Project Settings | [x] | Alt+Enter, ProjectSettingsModal (name, notes, sample rate, bit depth, tempo, time sig) | -| Undo / Redo | [x] | Ctrl+Z / Ctrl+Shift+Z, CommandManager pattern, 50-step history | -| Undo History Window | [x] | Ctrl+Alt+Z toggle, UndoHistoryPanel in View menu, floating panel | -| Metronome Toggle | [x] | Toggle in TransportBar, configurable accents/volume | -| Auto-Crossfade | [x] | Toggle in MainToolbar and View menu, auto-applies fadeIn/fadeOut on clip overlap | -| Snap Options | [x] | Snap toggle + grid size (bar/beat/half/quarter) in MainToolbar and View menu | -| Locking | [x] | Per-clip lock (prevents drag/resize/delete), lock icon overlay, context menu toggle | - -### Remaining Toolbar Features - -#### Auto-Crossfade Toggle -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** LOW - -When two audio clips overlap on the same track, automatically generate a crossfade at the intersection. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `autoCrossfade: boolean` (default: `true`) - - Add `toggleAutoCrossfade()` action - - Add `defaultCrossfadeLength: number` (default: `0.05` = 50ms) -2. **Timeline** (`Timeline.tsx`): - - In the clip drag/move handler, detect overlaps between clips on the same track - - When overlap detected and `autoCrossfade` enabled: - - Calculate overlap region - - Set `fadeOut` on the earlier clip and `fadeIn` on the later clip to match the overlap duration - - Render crossfade region visually (X-shaped lines between the two fade curves) -3. **Toolbar** (`MainToolbar.tsx`): - - Add auto-crossfade toggle button (icon: two overlapping waveforms) -4. **Backend**: No changes needed — fades already applied in PlaybackEngine - -#### Locking System -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** None - -Prevent accidental edits to specific elements (clip position, fades, volume, track order). - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `lockSettings: { items: boolean; envelopes: boolean; timeSelection: boolean; markers: boolean }` (all default `false`) - - Add `globalLocked: boolean` (default: `false`) - - Add `toggleGlobalLock()` and `setLockSetting(key, value)` actions - - Add `locked: boolean` to `AudioClip` interface -2. **Timeline** (`Timeline.tsx`): - - Before drag/resize/fade operations, check if clip is locked or if global lock + item lock is enabled - - Show lock icon overlay on locked clips - - Prevent drag if `lockSettings.items` is true -3. **UI**: Lock button in MainToolbar with right-click context menu showing granular lock options -4. **Track Header**: Add lock icon to per-clip context menu - ---- - -## II. File Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| New Project | [x] | Ctrl+N | -| Open Project | [x] | Ctrl+O | -| Recent Projects | [x] | Submenu with last 10 projects, localStorage persistence | -| Save Project | [x] | Ctrl+S | -| Save Project As | [x] | Ctrl+Shift+S | -| Project Settings | [x] | Alt+Enter | -| Close Project | [x] | Ctrl+F4, prompts save if modified | -| Clean Current Project Directory | [ ] | **NOT IMPLEMENTED** | -| Batch File/Item Converter | [ ] | **NOT IMPLEMENTED** | -| Render | [x] | Ctrl+Alt+R, RenderModal with format/bounds/options | -| Render Queue | [ ] | **NOT IMPLEMENTED** | -| Quit | [x] | Ctrl+Q | -| New Project in Tab | [ ] | **NOT IMPLEMENTED** (requires MDI architecture) | -| Save New Version | [x] | Increments version suffix (project.s13 → project_v2.s13), File menu | -| Save Live Output to Disk | [ ] | **NOT IMPLEMENTED** | -| Consolidate/Export Tracks | [ ] | **NOT IMPLEMENTED** | -| Export Project MIDI | [ ] | **NOT IMPLEMENTED** | -| Recovery Mode (bypass FX on open) | [ ] | **NOT IMPLEMENTED** | -| Timestamped Backup (.s13-bak) | [x] | Auto-backup at configurable intervals, Preferences dialog toggle | - -### Remaining File Menu Features - -#### ~~Close Project~~ — IMPLEMENTED - -#### Clean Current Project Directory -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** MEDIUM - -Scan the project folder, identify audio files not referenced by any clip, and offer to delete them. - -**Implementation Plan:** -1. **Backend** (`MainComponent.cpp`): - - Add `cleanProjectDirectory(projectPath, referencedFiles[])` → returns `{orphanedFiles: string[], totalSize: number}` - - Walks the project directory, compares against referenced file list - - Add `deleteFiles(filePaths[])` → deletes the orphaned files -2. **Frontend**: - - Collect all `clip.filePath` values from store - - Call backend with project directory + referenced files - - Show modal listing orphaned files with checkboxes + total size - - Confirm button calls deleteFiles - -#### Render Queue -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** LOW - -Queue multiple render jobs and execute them sequentially. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `renderQueue: RenderJob[]` (each job = full render options object) - - Add `addToRenderQueue(options)`, `removeFromRenderQueue(index)`, `executeRenderQueue()` actions -2. **UI**: "Add to Queue" button in RenderModal alongside "Render" -3. **RenderQueuePanel.tsx**: New component showing queued jobs with status, progress, remove buttons -4. **Execution**: Process jobs sequentially, update progress per-job - -#### Timestamped Backup -**Priority:** MEDIUM | **Complexity:** LOW | **Backend:** LOW - -Auto-save .s13-bak files at configurable intervals. - -**Implementation Plan:** -1. **Store**: Add `autoBackupInterval: number` (default: 300000 = 5 min), `autoBackupEnabled: boolean` -2. **App.tsx**: `setInterval` that calls `saveProject()` to `projectPath + '.bak'` with timestamp -3. **Backend**: No changes — `saveProjectToFile` already works with any path -4. **Settings**: Add auto-backup toggle + interval to Project Settings modal - -#### Recovery Mode (Bypass FX on Open) -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -Open a project without loading any VST3 plugins (useful when a plugin causes crashes). - -**Implementation Plan:** -1. Add `loadProject(path, { bypassFX: true })` option -2. When `bypassFX` is true, skip the FX restoration loop in `deserializeProject()` -3. Add "Open Project (Safe Mode)" to File menu or hold Shift while opening - -#### Export Project MIDI -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** MEDIUM - -Export all MIDI clips as a single .mid file. - -**Implementation Plan:** -1. **Backend**: Use `MIDIClip::exportToMidiFile()` (already exists as skeleton) -2. **Frontend**: Collect all MIDI clips, serialize events, call backend export -3. Requires MIDI clip system to be fully operational first - ---- - -## III. Edit Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Undo | [x] | Ctrl+Z | -| Redo | [x] | Ctrl+Shift+Z | -| Undo History | [x] | Ctrl+Alt+Z toggle, floating panel in View menu | -| Cut | [x] | Ctrl+X, clips | -| Copy | [x] | Ctrl+C, clips | -| Paste | [x] | Ctrl+V, at playhead on selected track | -| Duplicate | [x] | Ctrl+D | -| Delete | [x] | Delete key | -| Select All | [x] | Ctrl+A (tracks) | -| Deselect All | [x] | Esc | -| Ripple Editing (Off/Per-Track/All) | [ ] | **NOT IMPLEMENTED** | -| Split at Cursor | [x] | S key, splits selected clips (or all under playhead) with undo/redo | -| Split at Time Selection | [ ] | **NOT IMPLEMENTED** | -| Crossfade Editor | [ ] | **NOT IMPLEMENTED** | -| Item Properties | [ ] | **NOT IMPLEMENTED** (no dedicated clip inspector) | -| Razor Editing | [ ] | **NOT IMPLEMENTED** | -| Select All Clips | [x] | Ctrl+Shift+A selects all clips across all tracks | -| Cut within Time Selection | [x] | Trims clips to selection bounds, stores in clipboard, removes content | -| Copy within Time Selection | [x] | Trims clips to selection bounds, stores in clipboard | -| Nudge Items | [x] | Arrow Left/Right (grid), Ctrl+Arrow (10ms fine) | -| Dynamic Split (Transient Detection) | [ ] | **NOT IMPLEMENTED** | - -### Remaining Edit Menu Features - -#### ~~Split at Cursor (S key)~~ — IMPLEMENTED - -#### Split at Time Selection -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** None - -Split clips at both edges of the time selection, isolating the selected region. - -**Implementation Plan:** -1. Same logic as split at cursor but applied at both `timeSelection.start` and `timeSelection.end` -2. Creates up to 3 clips per original clip (before, within, after selection) - -#### Ripple Editing -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** None - -When deleting or inserting clips, automatically shift downstream clips to close/open gaps. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `rippleMode: 'off' | 'per_track' | 'all_tracks'` (default: `'off'`) - - Add `setRippleMode(mode)` action -2. **Modify existing actions**: `deleteClip`, `splitClipAtPlayhead`, paste — when ripple is enabled: - - `'per_track'`: After delete, find all clips on same track with `startTime > deletedClip.endTime`, shift them left by deleted clip's duration - - `'all_tracks'`: Same as above but for ALL tracks - - For insert/paste: shift downstream clips right -3. **UI**: Three-state toggle button in Edit menu and MainToolbar (Off → Per Track → All Tracks) -4. **Visual**: Show ripple mode indicator in toolbar - -#### Razor Editing -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** None - -Draw rectangular "razor" areas over specific clips/automation without splitting. Can be moved, copied, deleted. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `razorEdits: Array<{ trackId: string; startTime: number; endTime: number }>` - - Add `addRazorEdit()`, `removeRazorEdit()`, `moveRazorEdit()`, `deleteRazorEditContent()` actions -2. **Timeline** (`Timeline.tsx`): - - Alt+Right-Drag to create razor areas (red/orange rectangles) - - Razor areas are per-track (can span partial clips) - - Delete key on razor area: removes the audio within the razor bounds (splits clips at boundaries, deletes middle) - - Drag razor area to move the enclosed audio -3. **Render**: "Razor edit areas" source option in RenderModal - -#### Crossfade Editor -**Priority:** LOW | **Complexity:** HIGH | **Backend:** None - -Dedicated modal for fine-tuning crossfade shape between two overlapping clips. - -**Implementation Plan:** -1. **New component** `CrossfadeEditor.tsx`: - - Shows both waveforms at the crossfade point - - Fade curve shape selector (linear, equal-power, S-curve, logarithmic) - - Asymmetric fade handles (in/out can have different shapes) - - Preview playback of the crossfade region -2. **Store**: Add `crossfadeShape` to clip fade properties -3. **Backend**: PlaybackEngine already applies linear fades — extend to support curve types via lookup table - -#### Item/Clip Properties Inspector -**Priority:** MEDIUM | **Complexity:** LOW | **Backend:** None - -Floating panel showing all properties of the selected clip (name, file, duration, offset, volume, fades, sample rate). - -**Implementation Plan:** -1. **New component** `ClipPropertiesPanel.tsx`: - - Display: file path, format, sample rate, channels, duration - - Editable: name, volume (dB), fade in/out (seconds), offset, start time - - Opens on F2 or double-click clip -2. **Store**: Add `showClipProperties: boolean` and `toggleClipProperties()` action - -#### Dynamic Split (Transient Detection) -**Priority:** LOW | **Complexity:** HIGH | **Backend:** HIGH - -Automatically split a clip at transient peaks (e.g., drum hits). - -**Implementation Plan:** -1. **Backend** (`AudioEngine.cpp`): - - Add `detectTransients(filePath, sensitivity, minGap)` → returns `double[]` (array of hit times) - - Use spectral flux or onset detection algorithm -2. **Frontend**: Call backend, get transient times, call `splitClipAtPlayhead()` for each -3. **UI**: Modal with sensitivity slider, preview transient markers on waveform - -#### ~~Nudge Items~~ — IMPLEMENTED - ---- - -## IV. View Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Master Track Visible | [ ] | **NOT IMPLEMENTED** (master exists in mixer only) | -| Mixer Control Panel (MCP) | [x] | Ctrl+M toggle | -| Routing Matrix | [ ] | **NOT IMPLEMENTED** | -| Track Wiring Diagram | [ ] | **NOT IMPLEMENTED** | -| Region/Marker Manager | [x] | Floating panel, click to navigate, rename/delete, View menu toggle | -| Media Explorer | [ ] | **NOT IMPLEMENTED** | -| Big Clock | [x] | Floating clock with time/beats format, color-coded by transport state | -| Video Window | [ ] | **NOT IMPLEMENTED** | -| Screensets/Layouts | [ ] | **NOT IMPLEMENTED** | -| Docker/Panel System | [ ] | **NOT IMPLEMENTED** | -| Zoom In/Out/Fit | [x] | Ctrl++/-/0 | -| Loop Toggle | [x] | L key | -| Snap/Grid Settings | [x] | Toggle + grid size submenu | -| Virtual MIDI Keyboard | [x] | Alt+B toggle | - -### Remaining View Menu Features - -#### Master Track in TCP -**Priority:** MEDIUM | **Complexity:** LOW | **Backend:** None - -Show master track at the bottom of the track control panel (TCP) with volume fader, mute, and FX button. - -**Implementation Plan:** -1. **New component** `MasterTrackHeader.tsx`: - - Master volume fader (horizontal), mute button, FX button (opens master FX chain) - - Fixed at bottom of track header list -2. **Store**: Add `showMasterTrack: boolean` (default: `true`), `toggleMasterTrack()` action -3. **View menu**: "Show Master Track (Ctrl+Alt+M)" toggle item - -#### Routing Matrix -**Priority:** LOW | **Complexity:** HIGH | **Backend:** HIGH - -Grid-based view where any track can route to any other track or hardware output. - -**Implementation Plan:** -1. **Backend** (`AudioEngine.cpp`): - - Implement send/bus routing — `TrackProcessor` already has a `sends` skeleton - - Add `createBus()`, `setTrackSend(trackId, busId, level, pan, preFader)` methods - - Add bus tracks to the processing graph -2. **New component** `RoutingMatrix.tsx`: - - Grid: rows = source tracks, columns = destination tracks/outputs - - Click cell to toggle send, drag to set level -3. **Store**: Add `buses: Bus[]`, send levels per track - -#### Region/Marker Manager -**Priority:** MEDIUM | **Complexity:** LOW | **Backend:** None - -Spreadsheet-like panel listing all markers and regions with batch editing. - -**Implementation Plan:** -1. **New component** `MarkerManager.tsx`: - - Table with columns: Name, Type (marker/region), Time/Start, End, Color - - Click to navigate to marker/region - - Edit name/color inline - - Batch delete selected - - Sort by time/name -2. **Store**: Already has full CRUD for markers and regions -3. **View menu**: "Region/Marker Manager" item - -#### Media Explorer -**Priority:** LOW | **Complexity:** HIGH | **Backend:** MEDIUM - -Integrated file browser for auditioning and importing audio files. - -**Implementation Plan:** -1. **Backend**: Add `browseDirectory(path)` → returns file list with metadata - - Add `previewAudioFile(path)` → plays file through monitor output -2. **New component** `MediaExplorer.tsx`: - - Directory tree browser (left panel) - - File list with waveform previews - - Preview playback button (tempo-matched if possible) - - Drag-and-drop files onto timeline - - Filter by format, search by name -3. **Store**: Add `mediaExplorerPath`, `mediaExplorerFiles`, browsing actions - -#### Big Clock -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -Large floating time/beat display. - -**Implementation Plan:** -1. **New component** `BigClock.tsx`: - - Large monospace font showing: bars:beats:ticks or hh:mm:ss.ms - - Toggle between time formats - - Project name, tempo, time signature display - - Resizable floating panel -2. **Store**: Add `showBigClock: boolean`, `bigClockFormat: 'time' | 'beats'` -3. **View menu**: "Big Clock" toggle item - ---- - -## V. Insert Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| New Track | [x] | Ctrl+T (audio), Ctrl+Shift+T (MIDI) | -| Virtual Instrument on New Track | [x] | Creates instrument track + opens plugin browser, Insert menu | -| Media File | [x] | Insert key, frontend complete | -| Empty MIDI Item | [x] | Insert menu, creates 4-beat empty MIDI clip at playhead on selected/first MIDI track | -| Empty Item | [ ] | **NOT IMPLEMENTED** | -| Click Source | [ ] | **NOT IMPLEMENTED** | -| Marker | [x] | M key at playhead | -| Marker with Name | [x] | Shift+M | -| Region | [x] | Shift+R from time selection | -| Time Signature/Tempo Change Marker | [ ] | **NOT IMPLEMENTED** | -| Track Spacer | [ ] | **NOT IMPLEMENTED** | -| Insert Multiple Tracks | [x] | Prompt for count and type, Insert menu | -| SMPTE LTC/MTC Timecode | [~] | SMPTE timecode display in TransportBar (24/25/29.97/30fps), no LTC generation | - -### Remaining Insert Menu Features - -#### Virtual Instrument on New Track -**Priority:** HIGH | **Complexity:** MEDIUM | **Backend:** LOW - -Macro: creates a new instrument track, opens plugin browser, loads selected VSTi, arms for MIDI. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `addInstrumentTrack()` action that: - 1. Creates track with type `'instrument'` - 2. Opens plugin browser filtered to instruments only - 3. On plugin select: calls `loadInstrument(trackId, vstPath)` (already exists in backend) - 4. Arms track for MIDI, enables input monitoring -2. **PluginBrowser.tsx**: Add `filterType: 'all' | 'effects' | 'instruments'` prop -3. **Insert menu**: "Virtual Instrument on New Track..." item -4. **Backend**: `loadInstrument()` already implemented - -#### Time Signature / Tempo Change Markers -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** MEDIUM - -Place tempo/time-signature changes at specific positions on the timeline, creating a tempo map. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `tempoMarkers: Array<{ id: string; time: number; bpm: number; timeSigNum: number; timeSigDen: number; curveType: 'instant' | 'linear' }>` - - Add `addTempoMarker(time, bpm, timeSig)`, `removeTempoMarker(id)`, `updateTempoMarker(id, updates)` actions - - Modify `getTempoAtTime(time)` utility to interpolate between markers -2. **Timeline** (`Timeline.tsx`): - - Render tempo markers as distinct icons on the ruler (different color from regular markers) - - Double-click to edit tempo/time-sig values -3. **Backend**: Update metronome and PlaybackEngine to read from tempo map instead of single global BPM -4. **Grid calculation**: `snapToGrid.ts` must account for variable tempo - -#### Insert Multiple Tracks -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -Dialog to create N tracks at once with naming pattern. - -**Implementation Plan:** -1. **Modal**: Input for count (1-100), track type dropdown, naming pattern (e.g., "Track %n") -2. **Store**: Loop `addTrack()` N times with generated names -3. **Insert menu**: "Insert Multiple Tracks..." item - -#### Track Spacer -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -Visual-only gap between tracks in the TCP for organization. - -**Implementation Plan:** -1. **Store**: Add `spacers: Array<{ id: string; afterTrackId: string; height: number }>` -2. **Track header list**: Render spacer dividers between tracks where configured -3. **Context menu**: "Insert Spacer Below" option on track headers - ---- - -## VI. Item (Clip) Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Item Properties (F2) | [x] | ClipPropertiesPanel — editable name, volume, fades, mute/lock, F2 toggle | -| Source Properties | [ ] | **NOT IMPLEMENTED** | -| Quantize (audio to grid) | [ ] | **NOT IMPLEMENTED** | -| Pitch/Playback Rate (time stretch) | [ ] | **NOT IMPLEMENTED** | -| Takes (Explode/Implode) | [ ] | **NOT IMPLEMENTED** | -| Grouping | [ ] | **NOT IMPLEMENTED** | -| Normalize | [ ] | **NOT IMPLEMENTED** | -| Reverse | [ ] | **NOT IMPLEMENTED** | -| Dynamic Split | [ ] | **NOT IMPLEMENTED** | -| Mute Clip | [x] | U key, per-clip mute with visual feedback (dimmed + stripes), skipped in playback | -| Lock Clip | [x] | Per-clip lock toggle, prevents drag/resize/delete, lock icon overlay | -| Clip Color | [x] | Per-clip color via context menu with 10 preset colors | - -### Remaining Item Menu Features - -#### ~~Clip Mute~~ — IMPLEMENTED - -#### Clip Lock -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -Lock individual clips to prevent accidental movement/editing. - -**Implementation Plan:** -1. **Store**: Add `locked: boolean` to `AudioClip` interface (default: `false`) - - Add `toggleClipLock(clipId)` action -2. **Timeline**: Show lock icon on locked clips, prevent drag/resize/fade operations -3. **Context menu**: "Lock Clip" toggle option - -#### Clip Grouping -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** None - -Bind multiple clips together so editing one affects all grouped clips. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `clipGroups: Map` (groupId → clipIds) - - Add `groupClips(clipIds)`, `ungroupClips(groupId)` actions - - Add `groupId?: string` to `AudioClip` interface -2. **Timeline**: When moving/resizing a grouped clip, apply same transform to all clips in the group -3. **Visual**: Show matching colored border or group indicator on grouped clips -4. **Keyboard**: Ctrl+G to group selected clips, Ctrl+Shift+G to ungroup - -#### Takes (Explode/Implode) -**Priority:** LOW | **Complexity:** HIGH | **Backend:** None - -Stack multiple recording passes into a single clip container. Explode separates them to individual tracks. - -**Implementation Plan:** -1. **Store**: Add `takes: AudioClip[]` to `AudioClip` interface (multi-take container) - - Add `activeTakeIndex: number` (which take is active) - - Add `explodeTakes(clipId)`, `implodeTakes(clipIds)` actions -2. **Recording**: When loop-recording, stack passes as takes instead of separate clips -3. **Timeline**: Show take indicator badge, click to cycle takes -4. **Explode**: Creates one track per take at the same time position -5. **Implode**: Merges aligned clips on different tracks into a single multi-take clip - -#### Normalize Clip -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** HIGH - -Analyze clip's peak/LUFS and adjust gain to hit target level. - -**Implementation Plan:** -1. **Backend** (`AudioEngine.cpp`): - - Add `analyzeClipLevel(filePath, offset, duration)` → `{peakDB: number, rmsDB: number, lufs: number}` - - Add `normalizeClip(filePath, targetDB, mode: 'peak' | 'lufs')` → new file path or gain adjustment -2. **Frontend**: Context menu "Normalize..." → modal with target level and mode -3. **Simple approach**: Calculate required gain offset and apply to `clip.volume` instead of creating a new file - -#### Reverse Clip -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** HIGH - -Create a reversed copy of the audio file. - -**Implementation Plan:** -1. **Backend** (`AudioEngine.cpp`): - - Add `reverseAudioFile(filePath)` → returns path to reversed file - - Read source file, write samples in reverse order to new file -2. **Frontend**: Context menu "Reverse" → calls backend, replaces clip's filePath with reversed file -3. Wrap in undoable command - -#### Time Stretching / Pitch Shifting -**Priority:** LOW | **Complexity:** VERY HIGH | **Backend:** VERY HIGH - -Algorithmic time-stretch or pitch-shift clips without affecting the other parameter. - -**Implementation Plan:** -1. **Backend**: Integrate a time-stretch library (e.g., Rubber Band Library or SoundTouch) - - Add `timeStretchClip(filePath, stretchFactor)` → new file - - Add `pitchShiftClip(filePath, semitones)` → new file -2. **Store**: Add `playbackRate: number` and `pitchSemitones: number` to `AudioClip` -3. **PlaybackEngine**: Read samples at modified rate, use stretch algorithm -4. **UI**: Clip properties panel with rate/pitch controls -5. **Note**: This is one of the most complex features — consider deferring until core features are solid - ---- - -## VII. Track Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Insert New Track | [x] | Ctrl+T | -| Insert Multiple Tracks | [x] | Prompt for count and type, Insert menu and Track context | -| Render/Freeze Tracks | [x] | Freeze/unfreeze state, snowflake indicator, context menu toggle | -| Track Layout | [~] | Track height adjustable (Alt+Scroll), no saved layouts | -| Track Color | [x] | Color picker in track header | -| Envelopes/Automation | [x] | Volume/pan automation lanes, add/move/remove points, toggle visibility | -| Fixed Item Lanes | [ ] | **NOT IMPLEMENTED** | -| Free Item Positioning | [ ] | **NOT IMPLEMENTED** | -| Track Group Manager (VCA) | [ ] | **NOT IMPLEMENTED** | -| Track Templates | [x] | Save/load track configurations, context menu, localStorage persistence | - -### Remaining Track Menu Features - -#### Freeze/Render Tracks -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** HIGH - -Render a track's audio + FX to a new WAV file, bypass plugins, drastically reduce CPU. - -**Implementation Plan:** -1. **Backend** (`AudioEngine.cpp`): - - Add `freezeTrack(trackId, startTime, endTime)` → `{success: bool, frozenFilePath: string}` - - Uses same `fillTrackBuffer()` + FX processing as render, but for a single track - - Writes result to temp file in project directory - - Stores original FX state for unfreeze - - Add `unfreezeTrack(trackId)` → restores FX and removes frozen file -2. **Store** (`useDAWStore.ts`): - - Add `frozen: boolean` and `frozenFilePath?: string` to Track interface - - Add `freezeTrack(trackId)`, `unfreezeTrack(trackId)` actions - - When frozen: replace all clips with single frozen clip, bypass all FX - - When unfrozen: restore original clips and FX -3. **UI**: "Freeze Track" / "Unfreeze Track" in track context menu and Track menu -4. **Visual**: Show snowflake icon on frozen tracks, clips show "FROZEN" label - -#### Track Automation / Envelopes -**Priority:** HIGH | **Complexity:** VERY HIGH | **Backend:** HIGH - -Record and play back parameter changes over time (volume, pan, plugin parameters). - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `AutomationLane` interface: `{ id: string; trackId: string; paramId: string; paramName: string; points: AutomationPoint[]; visible: boolean; armed: boolean }` - - Add `AutomationPoint`: `{ time: number; value: number; curveType: 'linear' | 'bezier' | 'step' }` - - Add `automationLanes: AutomationLane[]` to Track - - Add `automationMode: 'read' | 'write' | 'touch' | 'latch' | 'trim'` per track - - Actions: `addAutomationPoint()`, `removeAutomationPoint()`, `moveAutomationPoint()`, `setAutomationMode()` -2. **Timeline** (`Timeline.tsx`): - - Render automation lanes below track (collapsible) - - Draw automation curves (lines between points, colored by parameter) - - Click to add points, drag to move, right-click to delete - - Show/hide automation per-parameter -3. **Backend** (`TrackProcessor.cpp`): - - In `processBlock()`, read automation values at current time - - Interpolate between points for smooth parameter changes - - Write mode: capture parameter changes as new points -4. **Automation Items**: Loopable containers of automation (advanced, defer) - -#### Fixed Item Lanes (Comping) -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** None - -Display multiple overlapping recordings as stacked lanes within a single track for easy comping. - -**Implementation Plan:** -1. **Store**: Add `laneMode: boolean` and `lanes: Lane[]` to Track - - `Lane`: `{ id: string; clips: AudioClip[]; playState: 'active' | 'muted' }` - - Add `enableLaneMode(trackId)`, `swipeComp(trackId, laneId, startTime, endTime)` actions -2. **Timeline**: When lane mode enabled, show stacked lanes within track height - - Each lane is a sub-row - - Click/swipe across lanes to select "active" regions - - Active region clips play, others are muted -3. **Recording**: New loop passes create new lanes instead of new clips -4. **Visual**: Lane borders, active region highlighting, comp result preview - -#### Track Group Manager (VCA-style) -**Priority:** LOW | **Complexity:** HIGH | **Backend:** MEDIUM - -Group tracks so adjusting one fader proportionally adjusts all grouped tracks. - -**Implementation Plan:** -1. **Store** (`useDAWStore.ts`): - - Add `trackGroups: Array<{ id: string; name: string; leadTrackId: string; memberTrackIds: string[]; linkedParams: string[] }>` - - Add `createTrackGroup()`, `addToGroup()`, `removeFromGroup()` actions - - Linked params: volume, pan, mute, solo, record arm -2. **Logic**: When lead track's volume changes, calculate delta and apply to all members (preserving relative offsets) -3. **UI**: "Track Group Manager" dialog accessible from Track menu -4. **Visual**: Color-coded group indicators on track headers - ---- - -## VIII. Options Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Record Modes (Normal/Overdub/Replace) | [x] | Normal/overdub/replace, Options menu submenu, TransportBar indicator | -| Auto-Crossfade | [x] | Toggle in MainToolbar and View menu, auto-applies on clip overlap | -| Ripple Editing | [x] | Off/per-track/all-tracks, Options menu submenu | -| Metronome Settings | [x] | MetronomeSettings component with accents/volume | -| Snap/Grid Settings | [x] | Full snap system with grid sizes | -| Lock Settings | [ ] | **NOT IMPLEMENTED** | -| Themes | [ ] | **NOT IMPLEMENTED** (single dark theme) | -| Theme Adjuster | [ ] | **NOT IMPLEMENTED** | -| Preferences | [x] | PreferencesModal with tabs: General, Editing, Display, Backup (Ctrl+,) | -| Move Envelope Points with Items | [ ] | **NOT IMPLEMENTED** (no automation yet) | -| Mouse Modifiers | [ ] | **NOT IMPLEMENTED** | - -### Remaining Options Menu Features - -#### Record Modes -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** MEDIUM - -- **Normal**: Creates a new clip (current behavior) -- **Overdub (MIDI)**: Merge new MIDI notes with existing clip -- **Replace**: Overwrite existing audio/MIDI in the recorded region - -**Implementation Plan:** -1. **Store**: Add `recordMode: 'normal' | 'overdub' | 'replace'` (default: `'normal'`) -2. **Recording logic**: - - Normal: Current behavior (new clip created) - - Overdub: For MIDI tracks, merge new events into existing MIDI clip at the recorded time range - - Replace: Delete any existing clips/portions that overlap the recording range, then insert new recording -3. **UI**: Options menu "Record Mode" submenu with radio items -4. **TransportBar**: Small indicator showing current record mode - -#### Preferences Dialog -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** LOW - -Comprehensive settings beyond audio device configuration. - -**Implementation Plan:** -1. Expand `SettingsModal.tsx` or create `PreferencesModal.tsx` with tabs: - - **Audio**: Current device settings (already implemented) - - **Project**: Default sample rate, bit depth, tempo, time signature for new projects - - **Paths**: Default project directory, VST3 scan paths, FFmpeg location - - **Editing**: Default crossfade length, ripple mode, snap behavior - - **Appearance**: UI scale, color accent, track height defaults - - **Keyboard**: Shortcut display/customization (read-only initially) -2. **Store**: Add `preferences: Preferences` object, persist to localStorage -3. **Options menu**: "Preferences... (Ctrl+P)" item - -#### Theme System -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** None - -Support multiple color themes beyond the current dark theme. - -**Implementation Plan:** -1. **CSS**: Define theme variables in `index.css` using CSS custom properties (already partially done with `daw-*` tokens) -2. **Store**: Add `theme: 'dark' | 'light' | 'midnight' | 'custom'` -3. **ThemeAdjuster**: New component allowing customization of accent color, background intensity, text brightness -4. Apply theme by swapping CSS custom property values on `:root` - ---- - -## IX. Actions Menu (Command Palette) - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Action List (4000+ commands) | [x] | Command Palette (Ctrl+Shift+P) + Action Registry with 100+ actions | -| Custom Actions (Macros) | [ ] | **NOT IMPLEMENTED** | -| ReaScript (EEL2/Lua/Python) | N/A | Not applicable — Studio13 has NativeBridge instead | -| Recent Actions | [ ] | **NOT IMPLEMENTED** | - -### Remaining Actions Menu Features - -#### Command Palette / Action List -**Priority:** MEDIUM | **Complexity:** MEDIUM | **Backend:** None - -Searchable list of all available actions with their keyboard shortcuts. - -**Implementation Plan:** -1. **New file** `store/actionRegistry.ts`: - ```typescript - interface ActionDef { - id: string; - name: string; - category: string; - shortcut?: string; - execute: () => void; - } - ``` - - Register all existing actions (transport, editing, track, view, etc.) -2. **New component** `CommandPalette.tsx`: - - Searchable modal (Ctrl+Shift+P or backtick key) - - Filter by category (Transport, Edit, View, Track, Insert, etc.) - - Shows shortcut next to each action - - Recent actions section at top - - Keyboard navigation (arrow keys + Enter to execute) -3. **Store**: Add `showCommandPalette: boolean`, `recentActions: string[]` -4. **Actions menu**: "Show Action List", "Show Recent Actions" - -#### Custom Actions (Macros) -**Priority:** LOW | **Complexity:** HIGH | **Backend:** None - -Combine multiple actions into a single macro. - -**Implementation Plan:** -1. **Store**: Add `customActions: Array<{ id: string; name: string; steps: string[]; shortcut?: string }>` -2. **UI**: Macro editor — select actions from registry, order them, assign shortcut -3. **Execution**: Run steps sequentially via action registry - ---- - -## X. Help Menu - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| User Guide | [ ] | **NOT IMPLEMENTED** | -| Keyboard Shortcuts Reference | [x] | KeyboardShortcutsModal — searchable, categorized, F1 shortcut | -| About Dialog | [x] | About Studio13 in Help menu with version, tech stack info | -| Changelog | [ ] | **NOT IMPLEMENTED** | -| Check for Updates | [ ] | **NOT IMPLEMENTED** | -| Diagnostics | [ ] | **NOT IMPLEMENTED** | - -### Remaining Help Menu Features - -#### About Dialog -**Priority:** LOW | **Complexity:** LOW | **Backend:** None - -**Implementation Plan:** -1. **New component** `AboutModal.tsx`: App name, version, logo, credits, license info -2. **Help menu**: "About Studio13 (Ctrl+F1)" - -#### Keyboard Shortcuts Reference -**Priority:** MEDIUM | **Complexity:** LOW | **Backend:** None - -**Implementation Plan:** -1. **New component** `KeyboardShortcutsModal.tsx`: - - Categorized table of all shortcuts (sourced from action registry) - - Search/filter functionality - - Print-friendly layout -2. **Help menu**: "Keyboard Shortcuts (Shift+F1)" - ---- - -## XI. Render to File Modal - -### Source Options - -| REAPER Source | Studio13 Status | Notes | -|---|---|---| -| Master mix | [x] | Default render source | -| Stems (selected tracks) | [x] | Per-track rendering via stem:trackId source, frontend loops tracks | -| Master mix + stems | [x] | Renders master + each track as separate files | -| Selected media items | [ ] | **NOT IMPLEMENTED** | -| Selected media items via master | [ ] | **NOT IMPLEMENTED** | -| Region render matrix | [ ] | **NOT IMPLEMENTED** | -| Region render matrix via master | [ ] | **NOT IMPLEMENTED** | -| Razor edit areas | [ ] | **NOT IMPLEMENTED** | - -### Bounds Options - -| REAPER Bounds | Studio13 Status | Notes | -|---|---|---| -| Entire project | [x] | Uses project range or clip extent | -| Custom time range | [x] | Manual start/end entry | -| Time selection | [x] | Uses timeSelection from store | -| Project regions | [ ] | **NOT IMPLEMENTED** | -| Selected regions | [ ] | **NOT IMPLEMENTED** | - -### Output Configuration - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Output directory | [x] | Browse button | -| Filename | [x] | Editable text input | -| Dynamic wildcards ($project, $track, $region, $date) | [x] | $project, $track, $date, $time, $index with live preview | -| Secondary output format | [ ] | **NOT IMPLEMENTED** | - -### Audio Parameters - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Sample rate | [x] | Renders at device rate, then FFmpeg converts to target sample rate | -| Channels (Stereo/Mono) | [x] | Working — mono downmix supported | -| Multichannel (up to 128ch) | [ ] | **NOT IMPLEMENTED** | -| Resample mode quality | [ ] | **NOT IMPLEMENTED** | - -### Output Formats - -| Format | Studio13 Status | Notes | -|---|---|---| -| WAV | [x] | 16/24/32-bit | -| AIFF | [x] | Working | -| FLAC | [x] | Working | -| MP3 | [x] | Render to WAV then FFmpeg encode (128/192/256/320 kbps) | -| OGG Vorbis | [x] | Render to WAV then FFmpeg encode (quality 1-10) | -| RAW PCM | [ ] | **NOT IMPLEMENTED** | -| DDP | [ ] | **NOT IMPLEMENTED** | -| Video (FFmpeg) | [ ] | **NOT IMPLEMENTED** | - -### Metadata & Post-Processing - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Normalize | [x] | 2-pass normalize to prevent clipping | -| Add tail (reverb) | [x] | Configurable tail length in ms | -| Dither/Noise Shaping | [ ] | **NOT IMPLEMENTED** (UI checkbox exists, backend ignores) | -| Embed metadata (BWF/ID3/ISRC) | [ ] | **NOT IMPLEMENTED** | -| Offline render (full speed) | [x] | Default mode — uses all CPU | -| Online render (1x speed) | [ ] | **NOT IMPLEMENTED** | -| Add rendered items to project | [ ] | **NOT IMPLEMENTED** | -| True Peak Limiting | [ ] | **NOT IMPLEMENTED** | -| LUFS Normalization | [ ] | **NOT IMPLEMENTED** | - -### Remaining Render Features - -#### ~~Stems Export (Selected Tracks)~~ — IMPLEMENTED - -Backend supports `stem:TRACK_ID` source parameter to render a single track (skips master FX/volume/pan). Frontend loops through tracks, calling renderProject per track with wildcard filename resolution. Progress shows "Rendering stem N of M: TrackName...". - -#### ~~Dynamic Wildcard Filenames~~ — IMPLEMENTED - -`resolveWildcards()` function in RenderModal supports `$project`, `$track`, `$date`, `$time`, `$index`. Live preview shown below filename input. Wildcard reference tooltip displayed. - -#### ~~Sample Rate Conversion on Export~~ — IMPLEMENTED - -Backend renders at device rate, then FFmpeg post-processes with `-ar` flag for sample rate conversion. Supports all standard rates (44.1k, 48k, 88.2k, 96k, 192k). Combined with lossy encoding in a single FFmpeg call when applicable. - -#### ~~MP3/OGG Encoding~~ — IMPLEMENTED - -Backend renders to temp WAV, then FFmpeg converts (MP3 via libmp3lame, OGG via libvorbis). Temp WAV deleted after conversion. FFmpeg found via `findFFmpegExe()` searching relative to executable. Frontend provides bitrate selector for MP3 (128-320 kbps) and quality selector for OGG (1-10). - -#### Metadata Embedding -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** MEDIUM - -Embed BWF, ID3, or iXML metadata into rendered files. - -**Implementation Plan:** -1. **Backend**: Use JUCE's `WavAudioFormat` metadata writing (supports BWF chunks) - - For WAV: Write BWF description, originator, origination date, time reference - - For MP3: Write ID3v2 tags (title, artist, album, etc.) -2. **UI**: "Metadata..." button in RenderModal opening a tag editor -3. **Store**: Add `renderMetadata: { title, artist, album, description, isrc }` to render options - -#### Region Render Matrix -**Priority:** LOW | **Complexity:** HIGH | **Backend:** HIGH - -2D matrix mapping tracks to regions. Each intersection produces a separate file. - -**Implementation Plan:** -1. **New component** `RegionRenderMatrix.tsx`: - - Grid: rows = tracks, columns = regions - - Check/uncheck intersections - - Preview filename for each intersection (using wildcards) -2. **Backend**: Loop through checked intersections, render each as separate file (solo track + set bounds to region) -3. **UI**: Accessible from RenderModal when source = "Region render matrix" - ---- - -## XII. Advanced / REAPER-Parity Features - -These are advanced features from REAPER that Studio13 will implement in later phases. - -| REAPER Feature | Studio13 Status | Notes | -|---|---|---| -| Multi-tab project (parallel sessions) | [ ] | **NOT IMPLEMENTED** — requires MDI architecture | -| 32-bit plugin bridging/firewalling | [ ] | **NOT IMPLEMENTED** — currently only 64-bit VST3 | -| SMPTE timecode generator/reader | [~] | SMPTE display in TransportBar (24/25/29.97/30fps), no LTC generation | -| Video playback window | [ ] | **NOT IMPLEMENTED** — needs FFmpeg video decoding | -| Custom toolbar creation | [ ] | **NOT IMPLEMENTED** — configurable toolbar editor | -| Mouse modifier customization matrix | [ ] | **NOT IMPLEMENTED** — per-context mouse modifier mapping | -| Scripting API / Extension system | [ ] | **NOT IMPLEMENTED** — Lua/JS scripting for user-created actions | -| DDP disc image export | [ ] | **NOT IMPLEMENTED** — CD mastering export format | -| Multichannel (128ch) rendering | [ ] | **NOT IMPLEMENTED** — surround/Ambisonics/Atmos | -| Free item positioning (items not locked to tracks) | [ ] | **NOT IMPLEMENTED** — items can float freely on timeline | - -### Remaining Section XII Features - -#### Multi-tab Project (MDI) -**Priority:** MEDIUM | **Complexity:** VERY HIGH | **Backend:** HIGH - -Open multiple projects simultaneously in tabs, each with independent transport/state. - -**Implementation Plan:** -1. **Architecture**: Wrap current App state in a `ProjectTab` context. Array of project states in a root store. -2. **Backend**: Support multiple AudioEngine instances or serialize/restore engine state per tab. -3. **UI**: Tab bar above MenuBar. Each tab holds its own track list, timeline, transport state. -4. **State**: `projectTabs: ProjectTab[]`, `activeTabIndex: number`. Each tab has full `DAWState`. - -#### 32-bit Plugin Bridging -**Priority:** LOW | **Complexity:** HIGH | **Backend:** HIGH - -Host 32-bit VST plugins in a separate process to prevent crashes and enable compatibility. - -**Implementation Plan:** -1. **Backend**: Create a bridge process (32-bit .exe) that loads 32-bit plugins. -2. **IPC**: Shared memory or pipes for audio buffer exchange between 64-bit host and 32-bit bridge. -3. **PluginManager**: Detect 32-bit plugins during scan, route them to bridge process. - -#### SMPTE Timecode -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** MEDIUM - -Generate/read SMPTE timecode for video sync and post-production workflows. - -**Implementation Plan:** -1. **Backend**: Implement SMPTE frame rate modes (24, 25, 29.97df, 30 fps). -2. **Store**: Add `timecodeMode` setting, `timecodeOffset` for start-of-day offset. -3. **UI**: Timecode display option in TransportBar (alongside bars:beats and seconds). - -#### Video Playback Window -**Priority:** MEDIUM | **Complexity:** HIGH | **Backend:** HIGH - -Display video synchronized to the timeline for scoring and post-production. - -**Implementation Plan:** -1. **Backend**: Use FFmpeg to decode video frames at current transport position. -2. **Frontend**: Floating video window component rendering frames via canvas/WebGL. -3. **Sync**: Backend sends video frame data on transport position change. -4. **Formats**: MP4, MOV, AVI via FFmpeg (already bundled in tools/). - -#### Custom Toolbar Creation -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** None - -Allow users to create and configure custom toolbars with selectable actions. - -**Implementation Plan:** -1. **Store**: `customToolbars: Array<{ id, name, buttons: Array<{ actionId, icon, label }> }>`. -2. **UI**: Toolbar editor modal — drag actions from action registry onto toolbar slots. -3. **Rendering**: Dynamic toolbar component that reads button definitions and renders icons. -4. **Persistence**: Save toolbar configs in project settings or user preferences. - -#### Mouse Modifier Customization -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** None - -Configure what happens on click/drag in different contexts based on modifier keys. - -**Implementation Plan:** -1. **Store**: `mouseModifiers: Map>` — e.g., `{ "clip_drag": { "ctrl": "copy", "alt": "move_vertical" } }`. -2. **UI**: Matrix editor in Preferences — rows = contexts (clip, track, timeline, envelope), columns = modifier combos. -3. **Timeline**: Replace hardcoded modifier checks with lookups into mouseModifiers map. - -#### Scripting API / Extension System -**Priority:** MEDIUM | **Complexity:** VERY HIGH | **Backend:** HIGH - -Enable user-written scripts (Lua or JS) to automate actions, create custom tools, and extend functionality. - -**Implementation Plan:** -1. **Backend**: Embed a Lua or QuickJS interpreter. Expose action registry, store state, and bridge functions. -2. **API surface**: `reaper.GetTrack()`, `reaper.InsertMedia()`, etc. — mimic ReaScript API where sensible. -3. **Frontend**: Script editor panel with syntax highlighting, run/stop buttons, console output. -4. **Security**: Sandbox scripts, limit filesystem access, provide opt-in permissions. - -#### DDP Disc Image Export -**Priority:** LOW | **Complexity:** MEDIUM | **Backend:** HIGH - -Export CD-ready DDP disc images with track markers and PQ codes. - -**Implementation Plan:** -1. **Backend**: Generate DDP file set (DDPID, DDPMS, image.dat) from rendered audio + region markers. -2. **Frontend**: DDP export option in Render modal when source = "Region render matrix". -3. **Markers**: Use regions as CD track boundaries, validate Red Book compliance (min 4s gap, 99 tracks max). - -#### Multichannel Rendering -**Priority:** LOW | **Complexity:** HIGH | **Backend:** HIGH - -Support surround sound and immersive audio formats (5.1, 7.1, Ambisonics, Atmos). - -**Implementation Plan:** -1. **Backend**: Extend AudioEngine to support >2 channel output. Track channel count per track. -2. **Render**: Multi-channel WAV writing (JUCE supports up to 256 channels in WAV). -3. **UI**: Channel configuration per track (mono/stereo/5.1/7.1/ambi), surround panner widget. -4. **Mixer**: Multi-channel meters and pan controls. - -#### Free Item Positioning -**Priority:** LOW | **Complexity:** HIGH | **Backend:** None - -Allow clips/items to float freely on the timeline without being locked to a specific track lane. - -**Implementation Plan:** -1. **Store**: Add `freePosition: { y: number }` to AudioClip — pixel offset within timeline. -2. **Timeline**: When free positioning enabled, clips render at their Y offset instead of track lane. -3. **Drag**: Clips can be dragged vertically anywhere, not just between track boundaries. -4. **Rendering**: Routing still based on track assignment, but visual position is independent. - ---- - -## XIII. Summary: What's Implemented vs. What's Remaining - -### Implementation Statistics - -| Category | Implemented | Remaining | Percentage | -|---|---|---|---| -| **Top Toolbar** | 10/10 | 0 | 100% | -| **File Menu** | 11/18 | 7 | 61% | -| **Edit Menu** | 21/21 | 0 | 100% | -| **View Menu** | 8/14 | 6 | 57% | -| **Insert Menu** | 11/13 | 2 | 85% | -| **Item/Clip Menu** | 5/12 | 7 | 42% | -| **Track Menu** | 7/10 | 3 | 70% | -| **Options Menu** | 7/11 | 4 | 64% | -| **Actions Menu** | 2/4 | 2 | 50% | -| **Help Menu** | 5/6 | 1 | 83% | -| **Render Source** | 3/8 | 5 | 38% | -| **Render Bounds** | 3/5 | 2 | 60% | -| **Render Formats** | 5/8 | 3 | 63% | -| **Render Metadata/Post** | 3/10 | 7 | 30% | -| **Advanced / REAPER-Parity** | 1/10 | 9 | 10% | -| **TOTAL** | **108/160** | **52** | **68%** | - -### Priority Breakdown of Remaining Features - -#### HIGH Priority (Core DAW Functionality) -1. ~~Split at Cursor (S key)~~ **DONE** -2. ~~Clip Mute~~ **DONE** -3. ~~Track Automation / Envelopes~~ **DONE** -4. ~~Stems Export (selected tracks render)~~ **DONE** -5. ~~Virtual Instrument on New Track~~ **DONE** -6. ~~Ripple Editing~~ **DONE** - -#### MEDIUM Priority (Workflow Enhancement) -1. ~~Split at Time Selection~~ **DONE** -2. ~~Razor Editing~~ **DONE** -3. ~~Record Modes (Normal/Overdub/Replace)~~ **DONE** -4. ~~Render Sample Rate Conversion~~ **DONE** -5. ~~MP3/OGG Encoding~~ **DONE** -6. ~~Region/Marker Manager~~ **DONE** -7. ~~Freeze/Render Tracks~~ **DONE** -8. ~~Dynamic Wildcard Filenames~~ **DONE** -9. ~~Clip Grouping~~ **DONE** -10. ~~Time Signature/Tempo Change Markers~~ **DONE** -11. ~~Preferences Dialog~~ **DONE** -12. ~~Command Palette / Action List~~ **DONE** -13. ~~Master Track in TCP~~ **DONE** -14. ~~Normalize Clip~~ **DONE** -15. Reverse Clip -16. ~~Undo History Panel (wire to View menu)~~ **DONE** -17. ~~Keyboard Shortcuts Reference~~ **DONE** -18. ~~Timestamped Backup (.s13-bak)~~ **DONE** -19. ~~Item/Clip Properties Inspector~~ **DONE** - -#### LOW Priority (Advanced / Nice-to-Have) -1. ~~Auto-Crossfade~~ **DONE** -2. ~~Locking System~~ **DONE** -3. Crossfade Editor -4. Dynamic Split (Transient Detection) -5. ~~Nudge Items~~ **DONE** -6. ~~Close Project~~ **DONE** -7. Clean Project Directory -8. Render Queue -9. Recovery Mode -10. ~~Fixed Item Lanes (Comping)~~ **DONE** -11. Track Group Manager (VCA) -12. ~~Track Templates~~ **DONE** -13. Media Explorer -14. ~~Big Clock~~ **DONE** -15. Theme System -16. Custom Actions (Macros) -17. Takes (Explode/Implode) -18. Time Stretching / Pitch Shifting -19. ~~About Dialog~~ **DONE** -20. Metadata Embedding -21. Region Render Matrix -22. ~~Insert Multiple Tracks~~ **DONE** -23. Track Spacer -24. Export Project MIDI -25. Secondary Output Format -26. ~~Select All Clips~~ **DONE** -27. ~~Cut/Copy within Time Selection~~ **DONE** -28. ~~Per-clip Color~~ **DONE** -29. Multi-tab Project (MDI) -30. 32-bit Plugin Bridging -31. ~~SMPTE Timecode~~ **DONE** (display only) -32. Video Playback Window -33. Custom Toolbar Creation -34. Mouse Modifier Customization -35. Scripting API / Extension System -36. DDP Disc Image Export -37. Multichannel Rendering -38. Free Item Positioning - ---- - -## XIV. Recommended Implementation Order - -### Phase 1: Essential Editing (HIGH priority) — COMPLETE -1. ~~**Split at Cursor**~~ — **DONE** (S key, undo/redo, context menu) -2. ~~**Clip Mute**~~ — **DONE** (U key, visual feedback, skipped in playback) -3. ~~**Undo History Panel**~~ — **DONE** (Ctrl+Alt+Z, View menu, floating panel) -4. ~~**Select All Clips**~~ — **DONE** (Ctrl+Shift+A) -5. ~~**Nudge Items**~~ — **DONE** (Arrow keys, Ctrl+Arrow for fine) -6. ~~**Close Project**~~ — **DONE** (Ctrl+F4, save prompt) - -### Phase 2: Render Completion — COMPLETE -4. ~~**MP3/OGG Encoding**~~ — **DONE** (FFmpeg encode, bitrate/quality selectors) -5. ~~**Stems Export**~~ — **DONE** (per-track rendering via stem:trackId, master+stems mode) -6. ~~**Sample Rate Conversion**~~ — **DONE** (FFmpeg resampling post-render) -7. ~~**Dynamic Wildcard Filenames**~~ — **DONE** ($project, $track, $date, $time, $index with live preview) - -### Phase 3: Workflow Features — COMPLETE -8. ~~**Ripple Editing**~~ — **DONE** (off/per-track/all-tracks modes, shifts downstream clips on delete) -9. ~~**Record Modes**~~ — **DONE** (normal/overdub/replace with overlap removal in replace mode) -10. ~~**Virtual Instrument on New Track**~~ — **DONE** (Insert menu item, creates instrument track + opens plugin browser) -11. ~~**Command Palette**~~ — **DONE** (Ctrl+Shift+P, searchable action list grouped by category) - -### Phase 4: Advanced Editing — COMPLETE -12. ~~**Razor Editing**~~ — **DONE** (Alt+drag creates per-track razor area, Delete removes content, red highlight) -13. ~~**Split at Time Selection**~~ — **DONE** (splits all clips at both edges of time selection) -14. ~~**Clip Grouping**~~ — **DONE** (Ctrl+G to group, Ctrl+Shift+G to ungroup, grouped clips select together) -15. ~~**Normalize Selected Clips**~~ — **DONE** (resets clip volume to 0 dB; Edit menu action) - -### Phase 5: Automation & Mixing — COMPLETE -16. ~~**Track Automation**~~ — **DONE** (volume/pan lanes, add/move/remove points, line rendering, double-click to add, drag to move) -17. ~~**Freeze Tracks**~~ — **DONE** (freeze/unfreeze state, visual indicator in TrackHeader, context menu toggle) -18. ~~**Master Track in TCP**~~ — **DONE** (virtual master track in MixerPanel with automation support) -19. ~~**Region/Marker Manager**~~ — **DONE** (floating panel, click to navigate, rename/delete, View menu toggle) - -### Phase 6: Polish & Advanced — COMPLETE -20. ~~**Tempo Map / Tempo Markers**~~ — **DONE** (add/remove/update tempo markers, getTempoAtTime for variable tempo) -21. ~~**Comping / Lanes**~~ — **DONE** (take lanes per track, promote/swap/delete takes, active take index) -22. ~~**Help Menu — Keyboard Shortcuts**~~ — **DONE** (F1 shows shortcuts dialog, About Studio13 info) -23. ~~**Help Menu — About & Command Palette**~~ — **DONE** (about dialog, Ctrl+Shift+P from Help menu) -24. **Remaining render features** — partial (metadata, region matrix still pending) - -### Phase 7: Frontend Polish & Completions — COMPLETE - -1. ~~**Auto-Crossfade**~~ — **DONE** (toggle in toolbar, auto-applies fadeIn/fadeOut on overlap) -2. ~~**Clip Lock & Per-clip Color**~~ — **DONE** (lock prevents edits, color via context menu) -3. ~~**Clip Properties Inspector (F2)**~~ — **DONE** (floating panel with editable properties) -4. ~~**Insert Multiple Tracks & Empty MIDI Item**~~ — **DONE** (Insert menu items) -5. ~~**Big Clock**~~ — **DONE** (floating time display with format toggle) -6. ~~**Cut/Copy within Time Selection**~~ — **DONE** (Edit menu actions) -7. ~~**Timestamped Backup & Track Templates**~~ — **DONE** (auto-backup timer, save/load templates) -8. ~~**Save New Version**~~ — **DONE** (increments version suffix in filename) -9. ~~**Keyboard Shortcuts Modal (F1)**~~ — **DONE** (searchable categorized modal from action registry) -10. ~~**SMPTE Timecode Display**~~ — **DONE** (TransportBar cycles time/beats/SMPTE) -11. ~~**Preferences Dialog (Ctrl+,)**~~ — **DONE** (tabbed modal: General, Editing, Display, Backup) - -### Phase 8: Platform & Extensibility - -1. **Multi-tab Project (MDI)** — parallel sessions in tabs -2. **Scripting API / Extension System** — Lua/JS automation and user-created actions -3. **Video Playback Window** — FFmpeg video decoding synced to transport -4. **Custom Toolbar Creation** — configurable toolbar editor -5. **Mouse Modifier Customization** — per-context modifier key mapping - -### Phase 9: Pro Audio & Compatibility - -1. **Multichannel Rendering** — surround/Ambisonics/Atmos (5.1, 7.1, 128ch) -2. **32-bit Plugin Bridging** — out-of-process bridge for legacy plugins -3. **Free Item Positioning** — clips float freely on timeline -4. **DDP Disc Image Export** — CD mastering format +# OpenStudio / Studio13-v3 Feature Inventory + +> **Last updated:** 2026-06-01 +> +> **Primary project format:** `.osproj` +> +> **Legacy project format:** `.s13` +> +> **Scope of this update:** current source tree, mounted UI, action registry, frontend bridge, backend native functions, and adjacent feature docs. +> +> **Note:** This file used to be a historical REAPER-parity tracker with stale counts. It is now a current feature inventory. Keep older planning docs for traceability, but use this file, `README.md`, and `docs/implemented_features.md` for public-facing feature claims. + +## Status Legend + +| Status | Meaning | +|---|---| +| Implemented | User-facing workflow has a UI/action and a store, bridge, or backend path. | +| Partial / Experimental | Real code exists, but the workflow is incomplete, hardware/runtime dependent, or not yet release-hardened. | +| Planned / Stub | UI state, bridge stubs, or planning notes exist, but the feature should not be advertised as complete. | + +## Latest Additions Since The Old Tracker + +| Area | Latest feature surface | +|---|---| +| AI runtime setup | In-app AI Tools setup with install, cancel, reset, status refresh, feature selection, background install notification, and platform-specific runtime plans. | +| AI music generation | ACE-Step 1.5 XL Turbo workflows for text-to-music and lyrics-plus-style generation. | +| AI audio generation | Stable Audio 3 Medium support for text-to-audio and source-conditioned workflows, including license-gated local model import. | +| Clip AI workflows | Create variation, inpaint selected range, and continue clip from a selected source audio clip. | +| Stem separation | Async stem separation with progress polling, cancellation, selectable stems, and result import to tracks. | +| Audio-to-MIDI | Convert an audio clip to a new MIDI track using polyphonic note extraction where the native Basic Pitch / ONNX path is available. | +| Detached editors | Detached mixer and detached MIDI editor window flows with UI snapshot publishing through the native bridge. | +| MIDI editor depth | Docked or windowed piano roll sessions, velocity/CC lanes, pitch bend controls, quantize, note transforms, and range editing. | +| Render/export | Dither path, secondary output, render queue, region render matrix, DDP export, batch converter, render-in-place, and add-render-to-project options. | +| Routing and mixing | Send/bus routing, routing matrix, sidechain assignment, channel output selection, stereo width, phase invert, pan law, and mixer snapshots. | +| Project delivery | Project compare, archive/unarchive, clean project directory, safe-mode open, project templates, and project tabs. | +| Video and sync | Video window plumbing, FFmpeg-backed video/audio extraction path, MTC input/output, MIDI clock sync state, and timecode settings. | +| Scripting and effects | Lua script editor/API, S13FX/JSFX audio effects, JSFX `@gfx` editor support, plugin A/B states, FX chain presets, and built-in FX presets. | +| Workflow customization | Command palette, shortcuts modal, screensets, theme editor, toolbar editor, custom actions/macros, mouse modifiers, and help surfaces. | + +## At A Glance + +| Area | Current support | +|---|---| +| Core engine | JUCE C++ audio engine with React/TypeScript WebView UI and synchronous `window.__JUCE__.backend` bridge. | +| Recording | Multitrack audio recording, MIDI recording, punch range, record modes, armed tracks, record-safe state, and input monitoring. | +| Editing | Clip move/trim/split, ripple edit, razor edit, time selection edits, fades, grouping, takes, slip edit, reverse, normalize, time stretch, and pitch shift. | +| MIDI | MIDI and instrument tracks, piano roll, virtual keyboard, MIDI import/export, MIDI output routing, quantize, transforms, CC lanes, and audio-to-MIDI. | +| Mixing | Mixer, detached mixer, sends, buses, routing matrix, master strip, channel strip EQ, metering, automation, mixer snapshots, and sidechains. | +| Plugins | VST3 hosting, CLAP/LV2 code paths, native editors, input/track/master/monitoring FX, presets, A/B states, MIDI learn, and built-in FX. | +| Pitch | Graphical pitch editor, YIN pitch analysis, note blobs, drift/vibrato tools, real-time pitch corrector, polyphonic detection, and ARA host plumbing. | +| AI | Optional local AI Tools runtime for stem separation, ACE-Step music generation, Stable Audio 3 audio generation, variation, inpaint, and continuation. | +| Render | WAV, AIFF, FLAC, MP3, OGG, stems, selected items, razor areas, regions, render queue, region matrix, DDP, batch conversion, dither, metadata, and secondary output. | +| Project tools | `.osproj` save/load, `.s13` legacy support, recent projects, templates, safe mode, autosave/backup, compare, archive, clean directory, missing media resolver. | +| Workflow | Command palette, keyboard shortcuts, menus, screensets, themes, toolbar editor, help overlay, getting started guide, big clock, and timecode settings. | +| Sync/media | MTC/MIDI clock plumbing, control surface manager, OSC support, MCU-style control surface paths, video window, and FFmpeg video extraction. | + +## Core Engine, Transport, And Recording + +### Implemented + +- Native JUCE audio engine with a React/TypeScript UI hosted in WebView2. +- Audio device settings for driver, device, input/output channels, sample rate, buffer size, and channel configuration. +- Sample-rate-aware clip playback and render mixing through the playback engine. +- Transport controls for play, pause, stop, record, seek, rewind, loop, set loop to time selection, and auto-scroll. +- Tempo, tap tempo, time signature, tempo markers, and metronome controls. +- Metronome accenting, volume, custom click/accent sounds, reset sounds, and render-to-file support. +- Multitrack audio recording with track arm, record-safe, input channel selection, monitoring, punch range, and record modes. +- MIDI recording with live preview and completed MIDI clip handoff. +- Background waveform peak cache and recording waveform previews. +- Real-time meter update event flow isolated from track arrays for render performance. + +### Partial / Experimental + +- External sync paths exist for MTC and MIDI clock, but device and studio integration still needs hardware validation. +- LTC output has a UI/bridge surface, but the backend explicitly treats generation as a stub. +- Live capture output has bridge/store state and should be treated as experimental until fully validated in-session. + +## Project, Files, And Media Management + +### Implemented + +- New, open, save, save as, close project, quit, and unsaved-changes confirmation flows. +- `.osproj` as the primary project extension with `.s13` legacy open/save support. +- Recent projects, startup recovery/diagnostics, and safe-mode project open with FX bypass. +- Project settings for name, notes, sample rate, bit depth, tempo, time signature, author/revision style metadata, and related state. +- Timestamped backup/autosave preferences. +- Project tabs, project templates, save-as-template, and new-from-template flows. +- Project compare with saved version. +- Session archive/unarchive backend paths. +- Media import through native dialogs, drag/drop, and audio/video import path. +- Missing media resolver. +- Media explorer surface, recent paths, and media import workflow. +- Clean project directory tool. +- Batch file converter modal. + +### Partial / Experimental + +- Media explorer audition/preview should be treated as limited until the backend preview engine is expanded. +- AAF/session interchange code paths exist, but AAF import is not a complete advertised workflow. + +## Arrangement And Clip Editing + +### Implemented + +- Konva timeline with ruler, grid, snap, zoom, scroll, playhead, waveform rendering, MIDI thumbnails, and time selection. +- Audio and MIDI clip creation, media import, drag/drop, move, resize, trim, and track-to-track moves. +- Multi-clip and multi-track selection, select all clips, select all tracks, and deselect all. +- Undo/redo through the command manager for editing operations. +- Cut, copy, paste, duplicate, delete, nudge, and fine nudge. +- Split at cursor and split at time selection. +- Cut/copy/delete within time selection and insert silence. +- Ripple editing modes: off, per-track, and all-tracks. +- Razor edit areas with content deletion and render source support. +- Clip mute, lock, color, volume, pan, fade in/out, fade shape, gain envelope, and clip properties panel. +- Auto-crossfade and a crossfade editor surface. +- Reverse clip, normalize selected clips, time stretch, and clip pitch shift. +- Dynamic split/transient split UI and execution path. +- Slip editing and free item positioning. +- Takes: add take, set active take, explode takes to tracks, and implode clips into takes. +- Track spacers, empty items, empty MIDI items, insert multiple tracks, and folder tracks. +- Clip launcher view. +- Quantize selected clips to grid. + +## MIDI And Instruments + +### Implemented + +- MIDI device enumeration, input opening, output routing, and MIDI panic. +- MIDI tracks and instrument tracks. +- Virtual instrument on new track and quick-add instrument track actions. +- MIDI clip storage, serialization, playback scheduling, import, export, and project MIDI export. +- Piano roll editor with docked and detached/windowed sessions. +- Note draw, edit, select, cut/copy/paste, delete, resize, move, and range operations. +- Velocity editing, CC lanes, pitch bend controls, visible lane preferences, and note inspector style surfaces. +- MIDI quantize using last settings, reset quantize, freeze quantize, and quantize test coverage. +- MIDI transforms: transpose, octave transpose, velocity scale, reverse notes, invert note pitches, and snap selected notes to scale. +- Virtual piano keyboard. +- MIDI input readiness checks before recording. +- Audio-to-MIDI conversion creates a generated MIDI track beside the source track, with undo support. +- Built-in instrument paths for basic synth, piano, drums, and sampler-style state. + +### Partial / Experimental + +- Drum editor and media pool toggles/actions exist, but should be confirmed as mounted, complete workflows before being advertised heavily. +- Step sequencer/step input state exists in the store surface, but the full user workflow needs validation. + +## Mixing, Routing, Metering, And Automation + +### Implemented + +- Mixer panel, channel strips, master strip, master track in TCP, and detached mixer window. +- Track controls for volume, pan, mute, solo, arm, input monitoring, record-safe, channel count, playback offset, and output channels. +- Master volume, pan, mute, mono, and master automation state. +- Peak/RMS meters, master meter cluster, clipping state, and reset meter clip actions. +- Channel strip EQ modal and built-in channel EQ backend parameters. +- Sends with level, pan, enable, pre/post fader, and phase invert. +- Bus/group tracks and create-bus-from-selected-tracks. +- Routing matrix and track routing modal. +- Sidechain source assignment into plugins. +- Phase invert, stereo width, master send enable, pan law, and DC offset handling. +- Track groups / linked group parameters. +- Mixer snapshots with save, recall, delete, backend sync, and undo on recall. +- Track and master automation lanes with read, write, touch, latch, manual point editing, range replace, backend sync, and envelope manager. +- Move-envelopes-with-items option. +- Loudness meter, phase correlation, and spectrum data bridge paths. + +## Plugins, FX, And Scripting + +### Implemented + +- Plugin scanning and loading for VST3, with CLAP/LV2 code paths present. +- Native plugin editor window management. +- Input FX, track FX, master FX, and monitoring FX chains. +- Add, remove, bypass, reorder, and open editor flows for FX chains. +- Plugin parameters, preset load/save, state save/load, and plugin A/B compare. +- Plugin MIDI learn, mapping list, and clear mapping flows. +- FX chain presets for track, input, and master chains. +- Built-in EQ, compressor, gate, limiter, delay, reverb, chorus, saturator, pitch corrector, and instrument-style processors. +- Built-in FX preset save/load/delete. +- Built-in FX oversampling controls. +- S13FX / JSFX-style script effects. +- JSFX `@gfx` native editor support through `S13FXGfxEditor`. +- Lua script execution, script directory/listing, script editor UI, and console output. +- App-facing Lua API documentation in `docs/API.md`. + +### Partial / Experimental + +- CLAP and LV2 support should be described as code-path/plugin-format support until compatibility is validated with a plugin test matrix. +- 32-bit plugin bridge is currently a preference/toggle surface, not a completed out-of-process legacy plugin host. + +## Pitch, Analysis, And Audio Processing + +### Implemented + +- Monophonic pitch analysis with YIN contour and note segmentation. +- Graphical pitch editor with note blobs, pitch contour, piano grid, zoom, scroll, and lower-zone UI. +- Pitch editor tools for pitch, drift, vibrato, transition, draw/split style editing, selection, undo, and redo. +- Scale/key snapping, chromatic snap, correct-pitch macro, and scale detection support. +- Offline monophonic graphical pitch correction/apply path. +- Pitch preview, scrub preview, rendered preview segments, and route/status diagnostics. +- Real-time auto-tune style pitch corrector FX. +- Polyphonic detection and audio-to-MIDI extraction through the Basic Pitch / ONNX path where available. +- Audio analyzer, transient detection, and silent-region style analysis utilities. +- ARA host controller lifecycle and track/clip ARA status plumbing. + +### Partial / Experimental + +- Polyphonic pitch correction/resynthesis is less mature than monophonic pitch editing. Treat detection and MIDI extraction as stronger than polyphonic audio correction. +- Subjective pitch, formant, stem, and artifact quality always requires user audition. Harness diagnostics are not proof of audible quality. + +## AI And Assisted Audio + +### Implemented + +- Optional AI Tools runtime status with refresh, install, cancel, reset, selected feature install, requested feature routing, and setup progress. +- Modular AI Tools feature IDs for `stemSeparation` and `audioGeneration`. +- Hardware/runtime status including GPU support hints and model/runtime readiness. +- Platform runtime packaging plans for Windows DirectML/CUDA and Linux CUDA/ROCm style profiles. +- Stem separation workflow with selectable stems, progress polling, cancellation, and imported result clips/tracks. +- AI track type and AI track header controls. +- AI generation modal and AI workflow parameter fields. +- ACE-Step 1.5 XL Turbo model surface. +- Stable Audio 3 Medium model surface. +- AI workflows: + - Text to Music + - Lyrics + Style + - Text to Audio + - Create Variation + - Inpaint Selection + - Continue Clip +- Source-conditioned AI generation from selected clips, with time-selection requirements for inpaint. +- AI generation progress and cancellation. +- Stable Audio setup includes local snapshot selection and license acknowledgement state. + +### Partial / Experimental + +- AI generation quality and speed depend on installed optional runtimes, local hardware, available VRAM/RAM, model availability, and accepted model licenses. +- The core app intentionally does not bundle large AI model runtimes. + +## Render, Export, And Delivery + +### Implemented + +- Offline render through the same playback/FX path used by playback. +- Render sources for master, stems, selected tracks, selected media items, selected items through master, and razor edit areas. +- Region render matrix UI. +- Render bounds for entire project, custom range, time selection, project regions, and selected regions where the UI path is available. +- Output directory, filename, and wildcard filename resolution. +- Metadata state for rendered files. +- Formats: WAV, AIFF, FLAC, MP3, and OGG. +- Sample-rate conversion and lossy encoding through FFmpeg where needed. +- Mono/stereo output, bit depth/quality options, normalize, render tail, and include-metronome options. +- Dithered render path through `renderProjectWithDither`. +- Secondary output format and secondary bit depth. +- Online render and add-rendered-output-to-project UI state. +- Render queue panel and queue actions. +- Render clip in place and render track in place. +- Consolidate track. +- Freeze/unfreeze track state and related UI. +- DDP disc image export backend and modal. +- Batch converter modal. +- Export project MIDI. + +### Partial / Experimental + +- Region render matrix, DDP, batch conversion, online render, and add-to-project workflows should receive release smoke testing before high-confidence public release notes. +- Advanced broadcast metadata, immersive multichannel delivery, and full CD mastering validation remain specialist areas to test separately. + +## Workflow, UI, And Customization + +### Implemented + +- Central action registry used by menus, command palette, shortcut reference, and global shortcuts. +- Menu bar, main toolbar, transport bar, lower zone, mixer, timeline, and modal surfaces. +- Command palette. +- Keyboard shortcuts modal. +- Preferences modal with general, editing, display, and backup preferences. +- Help overlay and getting started guide. +- Big clock and timecode settings. +- Screensets/layout state with save/load actions. +- Theme presets, theme editor, custom theme overrides, theme import/export style state, and high-contrast support. +- Toolbar editor and custom toolbar state. +- Custom actions/macros. +- Mouse modifier preferences and reset. +- Toast notifications, modal safety guards, startup recovery app, and error boundary. +- Detachable panels for mixer and MIDI editor. + +## Sync, Control Surfaces, Video, And Pro-Audio Plumbing + +### Implemented / Present + +- Timecode display and timecode settings panel. +- MTC generator and receiver paths. +- MIDI clock sync source/status plumbing. +- MIDI control surface manager. +- OSC control surface support. +- MCU-style control surface support. +- Video window component. +- FFmpeg-backed video metadata/frame extraction and audio extraction path. +- Surround/channel layout and VBAP panner source modules. +- Track channel format and master channel format state. + +### Partial / Experimental + +- Video support is functional plumbing, not a full post-production video suite. +- External sync and control surface workflows should be tested with real devices. +- Surround/immersive workflows need dedicated validation before being advertised as mature. + +## File Types And Sidecar Assets + +| Type | Current role | +|---|---| +| `.osproj` | Primary project file. | +| `.s13` | Legacy project file support. | +| `.ostheme` | Current theme export/import target. | +| `.s13theme` | Legacy theme import support. | +| `.ospreset` | Built-in FX preset style. | +| `.ospeaks` | Current waveform peak cache sidecar. | +| `.s13peaks` | Legacy waveform peak cache sidecar support. | +| `.mid` / `.midi` | MIDI import/export. | +| `.wav`, `.aiff`, `.flac`, `.mp3`, `.ogg` | Render/export and media workflows. | + +## Current Caveats To Keep Honest + +| Feature | Caveat | +|---|---| +| Polyphonic pitch correction | Detection and MIDI extraction are present; polyphonic audio correction/resynthesis is still less mature. | +| CLAP/LV2 hosting | Code paths exist; plugin compatibility still needs broad validation. | +| 32-bit plugin bridge | Toggle/state exists; full legacy plugin bridge is not complete. | +| LTC output | Bridge surface exists; backend generation is stubbed. | +| Media explorer preview | Import/browse surface exists; audition preview is limited. | +| AAF import/interchange | Source code exists, but it should not be advertised as complete AAF support. | +| Video | FFmpeg-backed video plumbing exists; editing/post-production workflow maturity is still evolving. | +| AI generation | Optional runtime and hardware dependent; model licenses and local setup affect availability. | +| Audio quality claims | Pitch, formant, stem, and generation quality must be validated by listening, not just diagnostics. | + +## Source Cross-Reference + +- `README.md` - public overview and product positioning. +- `docs/implemented_features.md` - codebase feature audit and caveats. +- `frontend/src/store/actionRegistry.ts` - commands exposed to menus, shortcuts, and command palette. +- `frontend/src/App.tsx` - mounted panels, modals, lower zones, detached windows, and workflow surfaces. +- `frontend/src/services/NativeBridge.ts` - frontend-to-native feature boundary. +- `frontend/src/data/aiWorkflows.ts` - AI models, workflows, and parameters. +- `frontend/src/store/actions/*` - current Zustand action modules. +- `Source/MainComponent.cpp` - native bridge functions exposed to the frontend. +- `Source/AudioEngine.*`, `Source/PlaybackEngine.*`, `Source/TrackProcessor.*`, `Source/MIDIManager.*`, `Source/Pitch*`, `Source/StemSeparator.*`, `Source/AITrackEngine.*`, and related source files - backend feature implementation. diff --git a/docs/features.pdf b/docs/features.pdf new file mode 100644 index 0000000..938a748 Binary files /dev/null and b/docs/features.pdf differ diff --git a/docs/new_features.md b/docs/new_features.md index bd4b4ed..df9ee88 100644 --- a/docs/new_features.md +++ b/docs/new_features.md @@ -1,5 +1,7 @@ # Studio13 New Features Implementation Plan +> **Status:** Historical implementation plan. Several items in this document have since moved from "planned" to implemented or partial. For current public-facing support/feature claims, use `README.md`, `docs/implemented_features.md`, and the mounted UI/actions as the source of truth. + ## Current State Summary | Area | Status | Notes | diff --git a/docs/pr_footprint_cleanup_20260501.md b/docs/pr_footprint_cleanup_20260501.md index 96c4481..394efeb 100644 --- a/docs/pr_footprint_cleanup_20260501.md +++ b/docs/pr_footprint_cleanup_20260501.md @@ -9,7 +9,7 @@ ## Ignore Or Remove - `tmp_pitch_runs/` generated pitch regression runs. Pre-cleanup status showed 2951 untracked files in this directory. -- `tools/openstudio_ace_backend/vendor_runtime/` downloaded/generated ACE/Comfy runtime payloads. Pre-cleanup status showed 429 untracked files in this directory. The installer/probe code reconstructs this runtime from the pinned runtime plan and cache; it should not be committed. +- `tools/openstudio_ace_backend/vendor_runtime/` downloaded/generated ACE runtime payloads. Pre-cleanup status showed 429 untracked files in this directory. The installer/probe code reconstructs this runtime from the pinned runtime plan and cache; it should not be committed. - Generated pitch regression WAV/PNG/JSONL reports under `tests/fixtures/pitch-regression`. - Local debug capture output under `pitch_debug_captures/`. diff --git a/docs/stable_audio_3_integration_plan.md b/docs/stable_audio_3_integration_plan.md new file mode 100644 index 0000000..164cd30 --- /dev/null +++ b/docs/stable_audio_3_integration_plan.md @@ -0,0 +1,148 @@ +# Stable Audio 3 + ACE-Step Source-Audio Workflow Checklist + +## Summary + +- [x] Keep the current ACE model/version exactly as-is: `ACE-Step 1.5 XL Turbo`. +- [x] Add `Stable Audio 3 Medium` as a separate selectable model with its own runtime, setup, and params. +- [x] Add supported source-audio workflows for both models from the clip context menu. +- [x] Do not implement ACE Extract, Lego, or Complete: no UI entries, workflow IDs, backend branches, or disabled placeholders. + +## AI Track Flow + +- [x] Keep AI tracks prompt-first. +- [x] Add a compact model dropdown in the AI track header: + - `ACE-Step 1.5 XL Turbo` + - `Stable Audio 3 Medium` +- [x] Add the same model dropdown at the top of the AI track params modal. +- [x] Make model changes update the workflow dropdown and visible params form. +- [x] Limit AI track workflows to: + - ACE-Step: `Text to Music`, `Lyrics + Style` + - Stable Audio 3: `Text to Audio` +- [x] If the selected model is unavailable, turn Generate into a setup CTA focused on that model. + +## Clip Context Menu Flow + +- [x] Add an `AI Generation` submenu to the existing audio clip right-click menu, near `Separate Stems...`. +- [x] Show the submenu only for audio clips. +- [x] Add menu items: + - `Create Variation...` + - `Inpaint Selection...` + - `Continue Clip...` +- [x] Disable `Inpaint Selection...` unless the current time selection overlaps the clicked clip. +- [x] Open a new `AI Clip Generation` modal from each menu item. +- [x] Include source clip name, source track name, and duration/range summary in the modal. +- [x] Limit modal model choices to models that support the selected source workflow. +- [x] Include workflow-specific params, setup/status block, and Generate/Cancel footer. +- [x] Do not show ACE Extract/Lego/Complete anywhere. + +## Output Placement + +- [x] Make source-audio workflows nondestructive by default. +- [x] `Create Variation...`: create a new audio track directly below the source track named `AI Variation - `, aligned to the source clip start. +- [x] `Inpaint Selection...`: create a new audio track directly below the source track named `AI Inpaint - `, aligned to the source clip start. +- [x] `Continue Clip...`: insert only the generated tail at `sourceClip.startTime + sourceClip.duration`. +- [x] Place continuation on the source track when clear; otherwise create `AI Continuation - ` below the source track. +- [x] Make all generated track/clip additions undo-tracked. + +## Workflow + Params Registry + +- [x] Replace the single ACE-only workflow registry with a model-aware registry. +- [x] Add model IDs: + - `ace-step-v15-xl-turbo` + - `stable-audio-3-medium` +- [x] Add workflow IDs: + - `text-to-music` + - `lyrics-style` + - `text-to-audio` + - `variation` + - `inpaint-selection` + - `continue-clip` +- [x] Declare supported model IDs, surface, params schema, and source requirements per workflow. +- [x] Add ACE source params: prompt, lyrics, seed, duration/extension duration, inference steps, guidance/cfg, `audio_cover_strength`, repaint start/end. +- [x] Add Stable Audio params: prompt, negative prompt, seed, duration/extension duration, steps, cfg scale, source strength/noise amount, inpaint range. +- [x] Add Stable Audio Advanced LoRA inference fields for optional `.safetensors` path and strength. + +## Frontend State + UI + +- [x] Add `aiMusicModelId` to AI tracks and default existing projects to ACE-Step. +- [x] Add modal state for clip generation source, workflow, model, params, and validation/status. +- [x] Build `AIClipGenerationModal` as a sibling to `StemSeparationModal`. +- [x] Reuse current dark theme tokens, compact controls, existing modal/footer layout, and existing UI components. +- [x] Convert time selection to clip-relative inpaint range: + - `rangeStart = max(timeSelection.start, clip.startTime) - clip.startTime` + - `rangeEnd = min(timeSelection.end, clipEnd) - clip.startTime` + - disable when `rangeEnd <= rangeStart` + +## Backend + Bridge + +- [x] Extend generation start API to `startAIGeneration(trackId, modelId, workflowId, paramsJSON)`. +- [x] Preserve old three-argument behavior by treating it as ACE-Step. +- [x] Enforce one global generation job at a time for v1. +- [x] Refactor `AITrackEngine` into provider routing. +- [x] Keep ACE provider on the existing persistent ACE worker. +- [x] Add a separate Stable Audio provider/runtime. +- [x] Include source file path, clip offset, clip duration, source track/clip IDs, inpaint range, or extension length in source params. +- [x] Prepare exact source-segment temp WAV before inference so trimmed clips behave correctly. +- [x] Map ACE Variation to Cover with `src_audio` and `audio_cover_strength`. +- [x] Map ACE Inpaint to Repaint with `src_audio`, `repainting_start`, and `repainting_end`. +- [x] Map ACE Continue to Repaint at the source clip end and crop the output to the generated tail before import. +- [x] Map Stable Audio Variation, Inpaint, and Continue to its source-conditioned workflows. +- [x] Add optional `modelId`, `workflowId`, and `sourceClipId` to progress payloads. + +## Setup Flow + +- [x] Add a music model section under AI Tools Setup > Audio Generation. +- [x] Keep ACE setup unchanged. +- [x] Add Stable Audio 3 Medium setup as strict opt-in. +- [x] Show Stability/Gemma license notice and require explicit checkbox confirmation. +- [x] Show `Powered by Stability AI` attribution when Stable Audio is selected or used. +- [x] Use manual Hugging Face download/import, not stored HF credentials. +- [x] Add `Open Hugging Face Model Page`. +- [x] Show required folder layout help text. +- [x] Add `Proceed with Setup` folder picker and validation. +- [x] Validate Stable Audio folder files: + - `model.safetensors` + - `model_config.json` + - `LICENSE.md` + - `LICENSE_GEMMA.md` + - `NOTICE` + - `t5gemma-b-b-ul2/model.safetensors` + - `t5gemma-b-b-ul2/config.json` + - `t5gemma-b-b-ul2/tokenizer.json` + - `t5gemma-b-b-ul2/tokenizer.model` + - `t5gemma-b-b-ul2/tokenizer_config.json` + - `t5gemma-b-b-ul2/special_tokens_map.json` +- [x] Validate the initial local target `C:\Users\srvds\Downloads\stable_audio_3`. +- [x] Copy/import the snapshot into a managed OpenStudio model folder. +- [x] Keep Stable Audio runtime separate from ACE. + +## Test + Visual Harness + +- [x] Add unit tests for model-aware workflow filtering, default params, normalization, inpaint range conversion, disabled inpaint, and no unsupported workflow IDs. +- [x] Add store tests for model changes, undoable generated source outputs, and continuation placement. +- [x] Add installer/probe tests for Stable Audio folder validation and license confirmation. +- [x] Add `tools/ai-generation-ui-harness.mjs`. +- [x] Harness scenario: `ai-track-model-selector`. +- [x] Harness scenario: `clip-context-ai-menu`. +- [x] Harness scenario: `ai-clip-generation-modal`. +- [x] Harness scenario: `stable-audio-setup`. +- [x] Harness checks desktop and compact screenshots. +- [x] Harness checks text overflow, viewport bounds, menu placement, and theme alignment. +- [x] Harness writes `qa/ai-generation//screenshots` and `qa/ai-generation//report.json`. + +## Verification + +- [x] Run `npx tsc --noEmit`. +- [x] Run the frontend visual harness. +- [x] Build frontend assets into `frontend/dist`. +- [x] Run `cmake --build build --config Debug`. +- [x] Stop Codex-started dev servers and leave port `5173` free. +- [ ] Manual check: ACE text generation from AI track. +- [ ] Manual check: ACE lyrics generation from AI track. +- [ ] Manual check: ACE clip variation. +- [ ] Manual check: ACE inpaint with overlapping time selection. +- [ ] Manual check: ACE continuation. +- [ ] Manual check: Stable Audio setup using `C:\Users\srvds\Downloads\stable_audio_3`. +- [ ] Manual check: Stable Audio text generation from AI track. +- [ ] Manual check: Stable Audio variation, inpaint, and continuation. +- [ ] Verify generated WAVs import, appear at expected positions, play back, and can be undone. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7f5fc2e..22b7691 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -24,6 +24,7 @@ import { startMidiEditorUISync, } from "./utils/midiEditorWindowSync"; import { maybeRunPitchRegressionDriver } from "./utils/pitchRegressionDriver"; +import { shouldAutoStopPlayback } from "./utils/transportAutoStop"; import { Button } from "./components/ui"; import { Timeline } from "./components/Timeline"; import { TimelineRuler } from "./components/TimelineRuler"; @@ -82,6 +83,7 @@ const TimecodeSettingsPanel = React.lazy(() => import("./components/TimecodeSett const HelpOverlay = React.lazy(() => import("./components/HelpOverlay").then(m => ({ default: m.HelpOverlay }))); const GettingStartedGuide = React.lazy(() => import("./components/GettingStartedGuide").then(m => ({ default: m.GettingStartedGuide }))); const StemSeparationModal = React.lazy(() => import("./components/StemSeparationModal")); +const AIClipGenerationModal = React.lazy(() => import("./components/AIClipGenerationModal")); const AiToolsSetupModal = React.lazy(() => import("./components/AiToolsSetupModal")); import { DndContext, @@ -171,6 +173,7 @@ function App() { masterAutomationLanes, showMasterAutomation, showStemSeparation, + showAIClipGeneration, showAiToolsSetup, closeAiToolsSetup, hydrateRecentProjects, @@ -246,6 +249,7 @@ function App() { masterAutomationLanes: state.masterAutomationLanes, showMasterAutomation: state.showMasterAutomation, showStemSeparation: state.showStemSeparation, + showAIClipGeneration: state.showAIClipGeneration, showAiToolsSetup: state.showAiToolsSetup, closeAiToolsSetup: state.closeAiToolsSetup, hydrateRecentProjects: state.hydrateRecentProjects, @@ -391,11 +395,10 @@ function App() { const previousAiToolsInstallInProgressRef = useRef(aiToolsStatus.installInProgress); const hasNativeMusicProfile = (aiToolsStatus.musicGenerationAvailableProfiles ?? []).length === 0 - || (aiToolsStatus.musicGenerationAvailableProfiles ?? []).includes("native-xl-turbo"); + || (aiToolsStatus.musicGenerationAvailableProfiles ?? []).includes("ace-diffusers"); const previousAiToolsFullReadyRef = useRef(Boolean( aiToolsStatus.musicGenerationReady && aiToolsStatus.musicGenerationLayoutValid - && (aiToolsStatus.musicGenerationPerformanceReady ?? true) && hasNativeMusicProfile, )); const [aiToolsInstallStatusStale, setAiToolsInstallStatusStale] = useState(false); @@ -475,7 +478,6 @@ function App() { currentState === "ready" && aiToolsStatus.musicGenerationReady && aiToolsStatus.musicGenerationLayoutValid - && (aiToolsStatus.musicGenerationPerformanceReady ?? true) && hasNativeMusicProfile, ); const currentPartialReady = Boolean( @@ -483,7 +485,6 @@ function App() { && ( !aiToolsStatus.musicGenerationReady || !aiToolsStatus.musicGenerationLayoutValid - || !(aiToolsStatus.musicGenerationPerformanceReady ?? true) || !hasNativeMusicProfile ), ); @@ -509,7 +510,7 @@ function App() { } else if (currentPartialReady && readyStateChanged) { showToast( (!hasNativeMusicProfile - ? "Stem separation is ready, but the OpenStudio ACE split profile is still missing required music-generation assets." + ? "Stem separation is ready, but the ACE-Step Diffusers backend is still missing required music-generation assets." : "") || aiToolsStatus.musicGenerationPerformanceStatusMessage @@ -729,7 +730,7 @@ function App() { } if (command === "transport.record") { - void state.record(); + void state.toggleRecord(); return; } @@ -833,6 +834,7 @@ function App() { let lastTime = performance.now(); let lastAutoUpdate = 0; // throttle automation value updates to ~30fps + let autoStopInFlight = false; let frameId: number; const loop = () => { @@ -844,6 +846,31 @@ function App() { if (currentState.transport.isPlaying) { let newTime = currentState.transport.currentTime + dt; + const autoStopDecision = shouldAutoStopPlayback({ + tracks: currentState.tracks, + transport: currentState.transport, + metronomeEnabled: currentState.metronomeEnabled, + nextTime: newTime, + }); + if (autoStopDecision.shouldStop) { + if (!autoStopInFlight) { + autoStopInFlight = true; + if (autoStopDecision.stopTime !== null) { + currentState.setCurrentTime(autoStopDecision.stopTime); + } + console.log("[App] Auto-stopping silent playback", { + reason: autoStopDecision.reason, + stopTime: autoStopDecision.stopTime, + latestEndTime: autoStopDecision.bounds.latestEndTime, + }); + void currentState.stop().catch((error) => { + console.warn("[App] Auto-stop silent playback failed", error); + autoStopInFlight = false; + }); + } + return; + } + // Loop: wrap back to loopStart when reaching loopEnd const { loopEnabled, loopStart, loopEnd } = currentState.transport; if (loopEnabled && loopEnd > loopStart && newTime >= loopEnd) { @@ -898,9 +925,9 @@ function App() { useDAWStore.setState((current) => ({ transport: { ...current.transport, - isPlaying: backendPlaying, + isPlaying: backendPlaying || Boolean(current.recordSession), isPaused: false, - isRecording: backendPlaying ? current.transport.isRecording : false, + isRecording: backendPlaying || Boolean(current.recordSession) ? current.transport.isRecording : false, currentTime: backendPos, }, })); @@ -1943,6 +1970,13 @@ function App() { )} + {/* AI Clip Generation Modal */} + {showAIClipGeneration && ( + + + + )} + {/* Channel Strip EQ Modal */} {showChannelStripEQ && ( @@ -2041,6 +2075,7 @@ function App() { {/* AI Tools Background Install Popup */} {!showStemSeparation && + !showAIClipGeneration && !showAiToolsSetup && showAiToolsBackgroundInstallPopup && aiToolsStatus.installInProgress && ( diff --git a/frontend/src/__tests__/aiClipGenerationModal.test.ts b/frontend/src/__tests__/aiClipGenerationModal.test.ts new file mode 100644 index 0000000..f8f3917 --- /dev/null +++ b/frontend/src/__tests__/aiClipGenerationModal.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { ACE_STEP_MODEL_ID, STABLE_AUDIO_3_MODEL_ID } from "../data/aiWorkflows"; +import { buildAIClipGenerationRequestParams } from "../components/AIClipGenerationModal"; +import modalSource from "../components/AIClipGenerationModal.tsx?raw"; +import trackHeaderSource from "../components/AITrackHeader.tsx?raw"; +import workflowModalSource from "../components/AIWorkflowModal.tsx?raw"; + +describe("AI clip generation modal", () => { + it("requires direction prompts for Stable Audio source workflows", () => { + expect(modalSource).toContain("stableSourcePromptMissing"); + expect(modalSource).toContain("Stable Audio source workflows need a direction prompt."); + expect(modalSource).toContain("disabled={!sourceClip || (isModelReady && (inpaintSelectionMissing || stableSourcePromptMissing))}"); + }); + + it("does not render misleading ETA or remaining-time text", () => { + for (const source of [modalSource, workflowModalSource, trackHeaderSource]) { + expect(source).not.toContain("formatEtaLabel"); + expect(source).not.toContain("s left"); + expect(source).not.toContain("m ${seconds}s left"); + } + expect(modalSource).toContain("formatElapsedLabel(progress.elapsedMs)"); + expect(workflowModalSource).toContain("formatElapsedLabel(track.aiGenerationElapsedMs)"); + expect(trackHeaderSource).toContain("formatElapsedLabel(track.aiGenerationElapsedMs)"); + }); + + it("injects project timing into ACE source clip requests", () => { + const request = buildAIClipGenerationRequestParams({ + params: { prompt: "make a close variation", audio_cover_strength: 0.85 }, + modelId: ACE_STEP_MODEL_ID, + sourceTrack: { id: "track-source", name: "Source" }, + sourceClip: { + id: "clip-source", + filePath: "C:/audio/source.wav", + name: "Loop", + startTime: 4, + duration: 8, + offset: 1, + color: "#38bdf8", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + }, + workflowRange: null, + extensionDuration: 20, + transportTempo: 100, + timeSignature: { numerator: 4, denominator: 4 }, + }); + + expect(request).toMatchObject({ + bpm: 100, + timesignature: "4/4", + source: { + filePath: "C:/audio/source.wav", + clipOffset: 1, + clipDuration: 8, + sourceClipStartTime: 4, + }, + }); + }); + + it("leaves Stable Audio source clip requests model-specific", () => { + const request = buildAIClipGenerationRequestParams({ + params: { prompt: "make a close variation" }, + modelId: STABLE_AUDIO_3_MODEL_ID, + sourceTrack: { id: "track-source", name: "Source" }, + sourceClip: { + id: "clip-source", + filePath: "C:/audio/source.wav", + name: "Loop", + startTime: 4, + duration: 8, + offset: 0, + color: "#38bdf8", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + }, + workflowRange: null, + extensionDuration: 20, + transportTempo: 100, + timeSignature: { numerator: 4, denominator: 4 }, + }); + + expect(request).not.toHaveProperty("bpm"); + expect(request).not.toHaveProperty("timesignature"); + }); +}); diff --git a/frontend/src/__tests__/aiGenerationStore.test.ts b/frontend/src/__tests__/aiGenerationStore.test.ts new file mode 100644 index 0000000..2851a85 --- /dev/null +++ b/frontend/src/__tests__/aiGenerationStore.test.ts @@ -0,0 +1,134 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { STABLE_AUDIO_3_MODEL_ID } from "../data/aiWorkflows"; +import { nativeBridge } from "../services/NativeBridge"; +import { commandManager } from "../store/commands"; +import { createDefaultTrack, type AudioClip, useDAWStore } from "../store/useDAWStore"; + +const initialState = useDAWStore.getState(); + +function sourceClip(overrides: Partial = {}): AudioClip { + return { + id: "clip-source", + filePath: "C:/audio/source.wav", + name: "Source Loop", + startTime: 10, + duration: 5, + offset: 0, + color: "#4cc9f0", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + sampleRate: 44100, + sourceLength: 5, + ...overrides, + }; +} + +describe("AI generation store actions", () => { + beforeEach(() => { + commandManager.clear(); + useDAWStore.setState(initialState); + vi.spyOn(nativeBridge, "importMediaFile").mockResolvedValue({ + filePath: "C:/audio/generated.wav", + duration: 3, + sampleRate: 44100, + numChannels: 2, + format: "wav", + }); + vi.spyOn(nativeBridge, "addTrack").mockResolvedValue("track-added"); + vi.spyOn(nativeBridge, "removeTrack").mockResolvedValue(true); + vi.spyOn(nativeBridge, "addPlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "removePlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "refreshWaveformPeaks").mockResolvedValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + commandManager.clear(); + useDAWStore.setState(initialState); + }); + + it("resets incompatible AI track workflow params when the model changes", () => { + const aiTrack = createDefaultTrack("ai-track", "AI Track", "#7c3aed", "ai"); + useDAWStore.setState({ tracks: [aiTrack], canUndo: false, canRedo: false }); + + useDAWStore.getState().setAITrackModel("ai-track", STABLE_AUDIO_3_MODEL_ID); + + const updated = useDAWStore.getState().tracks[0]; + expect(updated.aiMusicModelId).toBe(STABLE_AUDIO_3_MODEL_ID); + expect(updated.aiWorkflow).toBe("text-to-audio"); + expect(updated.aiWorkflowParams).toMatchObject({ + prompt: "", + negative_prompt: "", + duration: 30, + }); + expect(useDAWStore.getState().canUndo).toBe(true); + + useDAWStore.getState().undo(); + expect(useDAWStore.getState().tracks[0].aiWorkflow).toBe("text-to-music"); + }); + + it("adds variation output as an undoable track below the source track", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "variation", + filePath: "C:/audio/generated.wav", + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(2); + expect(tracks[1].name).toBe("AI Variation - Source Loop"); + expect(tracks[1].clips[0].startTime).toBe(10); + expect(tracks[1].clips[0].duration).toBe(5); + expect(useDAWStore.getState().canUndo).toBe(true); + + useDAWStore.getState().undo(); + expect(useDAWStore.getState().tracks).toHaveLength(1); + }); + + it("places continuation on the source track when the tail range is clear", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "continue-clip", + filePath: "C:/audio/generated.wav", + extensionDuration: 8, + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(1); + expect(tracks[0].clips).toHaveLength(2); + expect(tracks[0].clips[1].startTime).toBe(15); + expect(tracks[0].clips[1].duration).toBe(8); + }); + + it("places continuation on a new track when the source tail range collides", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [ + sourceClip(), + sourceClip({ id: "clip-existing", name: "Existing", startTime: 16, duration: 2 }), + ]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "continue-clip", + filePath: "C:/audio/generated.wav", + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(2); + expect(tracks[1].name).toBe("AI Continuation - Source Loop"); + expect(tracks[1].clips[0].startTime).toBe(15); + }); +}); diff --git a/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts b/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts index eb29985..4330a8e 100644 --- a/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts +++ b/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import modalSource from "../components/AiToolsSetupModal.tsx?raw"; import bridgeSource from "../services/NativeBridge.ts?raw"; import storeSource from "../store/useDAWStore.ts?raw"; +import mainComponentSource from "../../../Source/MainComponent.cpp?raw"; +import stemSeparatorSource from "../../../Source/StemSeparator.cpp?raw"; describe("AI feature installer contract", () => { it("defaults the setup checklist to requested compatible features or stem separation", () => { @@ -12,7 +14,10 @@ describe("AI feature installer contract", () => { }); it("keeps incompatible Audio Generation disabled while Stem Separation remains selectable", () => { - expect(modalSource).toContain("!feature.compatible"); + expect(modalSource).toContain("function buildSetupCatalog"); + expect(modalSource).toContain("disabled = !item.compatible && !item.ready"); + expect(modalSource).toContain("title={disabled ? item.disabledReason : statusLabel(item)}"); + expect(modalSource).toContain("Hardware not supported"); expect(modalSource).toContain("Incompatible features are disabled and will not be installed."); expect(modalSource).toContain("supported GPU with at least 8 GB memory was not detected"); expect(modalSource).toContain("CPU-only machines are supported."); @@ -29,4 +34,55 @@ describe("AI feature installer contract", () => { expect(storeSource).toContain("options.requestedFeature ?? \"stemSeparation\""); expect(storeSource).toContain("selectedFeatures: [\"stemSeparation\"]"); }); + + it("exposes strict Stable Audio 3 manual import controls", () => { + expect(modalSource).toContain("STABLE_AUDIO_MODEL_URL"); + expect(modalSource).toContain("Open Hugging Face Model Page"); + expect(modalSource).toContain("Proceed with Setup"); + expect(modalSource).toContain("Use Downloads Folder"); + expect(modalSource).toContain("Cancel Setup"); + expect(modalSource).toContain("stableAudioSelectedFolder"); + expect(modalSource).toContain("modelId: STABLE_AUDIO_3_MODEL_ID"); + expect(modalSource).toContain("stableAudioLicenseAccepted"); + expect(modalSource).toContain("LICENSE_GEMMA.md"); + expect(bridgeSource).toContain("stableAudioModelPath?: string"); + expect(bridgeSource).toContain("stableAudioLicenseAccepted?: boolean"); + expect(bridgeSource).toContain("modelId?: AiMusicModelId"); + expect(mainComponentSource).toContain(".withNativeFunction (\"browseForFolder\""); + expect(mainComponentSource).toContain("canSelectDirectories"); + }); + + it("validates Stable Audio 3 folder layout and license before import", () => { + expect(stemSeparatorSource).toContain("model.safetensors"); + expect(stemSeparatorSource).toContain("t5gemma-b-b-ul2/tokenizer.model"); + expect(stemSeparatorSource).toContain("Stable Audio 3 setup requires accepting"); + expect(stemSeparatorSource).toContain("stable_audio_model_path_required"); + expect(stemSeparatorSource).toContain("stable_audio_model_layout_invalid"); + expect(stemSeparatorSource).toContain("stable_audio_import_requested"); + expect(stemSeparatorSource).toContain("stable_audio_command_started"); + expect(stemSeparatorSource).toContain("Stable Audio 3 dependencies can take several minutes to install."); + expect(stemSeparatorSource).toContain("stable_audio_command_cancelled"); + expect(stemSeparatorSource).toContain("copyDirectoryTo(destination)"); + }); + + it("keeps Stable Audio runtime separate from the ACE runtime", () => { + expect(stemSeparatorSource).toContain("getStableAudioRuntimeRoot"); + expect(stemSeparatorSource).toContain("stable-audio-runtime"); + expect(stemSeparatorSource).toContain("Installing Stable Audio 3 CUDA PyTorch runtime"); + expect(stemSeparatorSource).toContain("git+https://github.com/Stability-AI/stable-audio-3.git"); + expect(stemSeparatorSource).toContain("stable_audio_flash_attention_skipped"); + expect(stemSeparatorSource).toContain("PyTorch attention fallback"); + }); + + it("keeps Stable Audio install options intact through the store action", () => { + expect(storeSource).toContain("const isStableAudioImport = options.modelId === STABLE_AUDIO_3_MODEL_ID"); + expect(storeSource).toContain("...options"); + expect(storeSource).toContain("Stable Audio 3 setup is starting in the background."); + }); + + it("sanitizes internal ACE runtime paths from setup UI copy", () => { + expect(modalSource).toContain("function sanitizeSetupMessage"); + expect(modalSource).toContain("ACE-Step runtime files are missing. Repair or reinstall ACE-Step Audio Generation setup."); + expect(modalSource).toContain("displayActivityLines"); + }); }); diff --git a/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts b/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts index 3e9abef..4bf536b 100644 --- a/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts +++ b/frontend/src/__tests__/aiToolsSetupModalWindowsLock.test.ts @@ -6,7 +6,7 @@ describe("AI tools setup modal Windows lock guidance", () => { expect(modalSource).toContain("runtime_locked_rebuild_failed"); expect(modalSource).toContain("runtime_rebuild_remove_failed"); expect(modalSource).toContain("runtime-only rebuild"); - expect(modalSource).toContain("Retry keeps the downloaded stem models and ACE-Step checkpoints in place."); + expect(modalSource).toContain("Retry keeps the downloaded stem models and ACE-Step Diffusers cache in place."); expect(modalSource).toContain("Reset AI Tools"); }); }); diff --git a/frontend/src/__tests__/aiWorkflowParams.test.ts b/frontend/src/__tests__/aiWorkflowParams.test.ts index 3b0e1e1..016a48f 100644 --- a/frontend/src/__tests__/aiWorkflowParams.test.ts +++ b/frontend/src/__tests__/aiWorkflowParams.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; import { + ACE_STEP_MODEL_ID, + AI_WORKFLOWS, + STABLE_AUDIO_3_MODEL_ID, + getAIModelsForWorkflow, + getAIWorkflowsForSurface, + getClipInpaintRange, getDefaultWorkflowParams, normalizeWorkflowParams, } from "../data/aiWorkflows"; @@ -17,14 +23,8 @@ describe("AI workflow params", () => { timesignature: "4/4", language: "en", keyscale: "C major", - generate_audio_codes: true, - cfg_scale: 2, - guidance_scale: 1, + inferenceSteps: 8, shift: 3, - temperature: 0.85, - top_p: 0.9, - top_k: 0, - min_p: 0, }); }); @@ -38,14 +38,8 @@ describe("AI workflow params", () => { timesignature: "3", language: "xx", keyscale: "not-a-key", - generate_audio_codes: "false", - cfg_scale: "4.5", - guidance_scale: "21", shift: "9.5", - temperature: "1.25", - top_p: "0.95", - top_k: "12", - min_p: "0.123", + inferenceSteps: "99", ignored: "value", }); @@ -54,23 +48,16 @@ describe("AI workflow params", () => { lyrics: "", seed: 42, bpm: 240, - duration: 5, + duration: 10, timesignature: "3/4", language: "en", keyscale: "C major", - generate_audio_codes: false, - inferenceSteps: 8, - cfg_scale: 4.5, - guidance_scale: 20, - shift: 5, - temperature: 1.25, - top_p: 0.95, - top_k: 12, - min_p: 0.123, + inferenceSteps: 24, + shift: 9.5, }); }); - it("preserves explicit musical metadata in the native parity request surface", () => { + it("preserves explicit musical metadata in the Diffusers request surface", () => { const normalized = normalizeWorkflowParams("text-to-music", { bpm: 170, duration: 191, @@ -83,21 +70,132 @@ describe("AI workflow params", () => { duration: 191, timesignature: "3/4", keyscale: "C# minor", - generate_audio_codes: true, }); }); - it("keeps generate_audio_codes enabled by default", () => { - const normalized = normalizeWorkflowParams("text-to-music", { - generate_audio_codes: true, - }); + it("keeps Diffusers-only ACE params visible by default", () => { + const normalized = normalizeWorkflowParams("text-to-music", {}); expect(normalized).toMatchObject({ bpm: 120, duration: 30, timesignature: "4/4", keyscale: "C major", - generate_audio_codes: true, + inferenceSteps: 8, + shift: 3, + }); + }); + + it("filters workflows by surface and selected model", () => { + expect(getAIWorkflowsForSurface("ai-track", ACE_STEP_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "text-to-music", + "lyrics-style", + ]); + expect(getAIWorkflowsForSurface("ai-track", STABLE_AUDIO_3_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "text-to-audio", + ]); + expect(getAIWorkflowsForSurface("clip-context", STABLE_AUDIO_3_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "variation", + "inpaint-selection", + "continue-clip", + ]); + }); + + it("uses Stable Audio defaults and clamps source workflow params", () => { + const defaults = getDefaultWorkflowParams("text-to-audio", STABLE_AUDIO_3_MODEL_ID); + expect(defaults).toMatchObject({ + prompt: "", + negative_prompt: "", + seed: -1, + duration: 30, + steps: 8, + cfg_scale: 1, + lora_path: "", + lora_strength: 1, }); + + const normalized = normalizeWorkflowParams("variation", { + steps: "999", + cfg_scale: "-4", + noise_amount: "-1", + lora_strength: "5", + }, STABLE_AUDIO_3_MODEL_ID); + + expect(normalized).toMatchObject({ + steps: 32, + cfg_scale: 0.1, + noise_amount: 0, + lora_strength: 2, + }); + }); + + it("keeps clip source workflow forms source-specific", () => { + const stableVariation = getDefaultWorkflowParams("variation", STABLE_AUDIO_3_MODEL_ID); + expect(stableVariation).toMatchObject({ + prompt: "Create a close musical variation that preserves the source clip's tempo, key, primary instrument, arrangement density, and mix character. Do not add vocals or unrelated instruments.", + negative_prompt: "unrelated instruments, unexpected vocals, full band arrangement unless present in the source, distorted, noisy, clipped, abrupt transition, low quality", + seed: -1, + noise_amount: 0.5, + steps: 8, + cfg_scale: 1, + }); + expect(stableVariation).not.toHaveProperty("duration"); + expect(stableVariation).not.toHaveProperty("extension_duration"); + expect(stableVariation).not.toHaveProperty("inpaint_start"); + + const stableContinue = getDefaultWorkflowParams("continue-clip", STABLE_AUDIO_3_MODEL_ID); + expect(stableContinue).toMatchObject({ + prompt: "Continue the same musical idea, matching the source clip's tempo, key, primary instrument, harmony, room tone, and mix. Do not add vocals or unrelated instruments unless requested.", + negative_prompt: "unrelated instruments, unexpected vocals, full band arrangement unless present in the source, distorted, noisy, clipped, abrupt transition, low quality", + seed: -1, + extension_duration: 8, + steps: 8, + cfg_scale: 1, + }); + expect(stableContinue).not.toHaveProperty("duration"); + expect(stableContinue).not.toHaveProperty("noise_amount"); + + const stableInpaint = getDefaultWorkflowParams("inpaint-selection", STABLE_AUDIO_3_MODEL_ID); + expect(stableInpaint.prompt).toBe("Replace the selected range naturally while matching the surrounding clip."); + expect(stableInpaint).not.toHaveProperty("duration"); + + const aceVariation = getDefaultWorkflowParams("variation", ACE_STEP_MODEL_ID); + expect(aceVariation).toMatchObject({ + prompt: "Create a close musical variation that preserves the source clip's tempo, key, primary instrument, and arrangement density. Do not add vocals or unrelated instruments.", + seed: -1, + audio_cover_strength: 0.85, + inferenceSteps: 8, + shift: 3, + }); + expect(aceVariation).not.toHaveProperty("lyrics"); + expect(aceVariation).not.toHaveProperty("duration"); + expect(aceVariation).not.toHaveProperty("extension_duration"); + }); + + it("converts timeline time selection into clip-relative inpaint range", () => { + expect(getClipInpaintRange({ start: 12, end: 16 }, { startTime: 10, duration: 8 })).toEqual({ + start: 2, + end: 6, + }); + expect(getClipInpaintRange({ start: 1, end: 4 }, { startTime: 10, duration: 8 })).toBeNull(); + expect(getClipInpaintRange(null, { startTime: 10, duration: 8 })).toBeNull(); + }); + + it("does not expose unsupported ACE-only workflow ids", () => { + const ids = AI_WORKFLOWS.map((workflow) => workflow.id); + expect(ids).not.toContain("extract"); + expect(ids).not.toContain("lego"); + expect(ids).not.toContain("complete"); + expect(ids).not.toContain("continuation"); + }); + + it("limits source workflow model choices to supporting models", () => { + expect(getAIModelsForWorkflow("variation").map((model) => model.id)).toEqual([ + ACE_STEP_MODEL_ID, + STABLE_AUDIO_3_MODEL_ID, + ]); + expect(getAIModelsForWorkflow("text-to-audio").map((model) => model.id)).toEqual([ + STABLE_AUDIO_3_MODEL_ID, + ]); }); }); diff --git a/frontend/src/__tests__/audioToMidiConversion.test.ts b/frontend/src/__tests__/audioToMidiConversion.test.ts new file mode 100644 index 0000000..5673e1c --- /dev/null +++ b/frontend/src/__tests__/audioToMidiConversion.test.ts @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { nativeBridge } from "../services/NativeBridge"; +import { audioToMidiActions, mapPolyNotesToMIDIEvents } from "../store/actions/audioToMidi"; +import { commandManager } from "../store/commands"; +import { createDefaultTrack, type AudioClip, useDAWStore } from "../store/useDAWStore"; +import actionRegistrySource from "../store/actionRegistry.ts?raw"; +import timelineSource from "../components/Timeline.tsx?raw"; + +const initialState = useDAWStore.getState(); + +function flushPromises() { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function sourceClip(overrides: Partial = {}): AudioClip { + return { + id: "audio-clip", + filePath: "C:/audio/source.wav", + name: "Source Loop", + startTime: 12, + duration: 5, + offset: 1, + color: "#4cc9f0", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + sampleRate: 44100, + sourceLength: 8, + ...overrides, + }; +} + +function analysis(notes: any[] = []) { + return { + clipId: "audio-clip", + sampleRate: 22050, + hopSize: 256, + pitchSalience: [], + salienceDownsampleFactor: 1, + notes, + }; +} + +describe("audio clip to MIDI conversion", () => { + beforeEach(() => { + commandManager.clear(); + useDAWStore.setState(initialState); + vi.spyOn(nativeBridge, "extractMidiFromAudio").mockResolvedValue(analysis()); + vi.spyOn(nativeBridge, "addTrack").mockResolvedValue("generated-track"); + vi.spyOn(nativeBridge, "setTrackType").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTrackMIDIClips").mockResolvedValue(true); + vi.spyOn(nativeBridge, "removeTrack").mockResolvedValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + commandManager.clear(); + useDAWStore.setState(initialState); + }); + + it("maps detected notes to clipped MIDI note events", () => { + const events = mapPolyNotesToMIDIEvents([ + { startTime: -0.5, endTime: 0.5, midiPitch: 60.2, velocity: 0.5 }, + { startTime: 4.8, endTime: 7, midiPitch: 64, velocity: 2 }, + { startTime: 1, endTime: 1.005, midiPitch: 65, velocity: 0.8 }, + { startTime: Number.NaN, endTime: 2, midiPitch: 67, velocity: 0.8 }, + ] as any, 5); + + expect(events).toEqual([ + { timestamp: 0, type: "noteOn", note: 60, velocity: 64 }, + { timestamp: 0.5, type: "noteOff", note: 60, velocity: 0 }, + { timestamp: 4.8, type: "noteOn", note: 64, velocity: 2 }, + { timestamp: 5, type: "noteOff", note: 64, velocity: 0 }, + ]); + }); + + it("creates one silent MIDI track below the source track with a matching clip duration", async () => { + vi.mocked(nativeBridge.extractMidiFromAudio).mockResolvedValue(analysis([ + { startTime: 0.25, endTime: 2, midiPitch: 60, velocity: 0.75 }, + { startTime: 4.5, endTime: 8, midiPitch: 67, velocity: 1 }, + ])); + + const sourceTrack = createDefaultTrack("source-track", "Audio", "#4cc9f0", "audio"); + sourceTrack.clips = [sourceClip()]; + const nextTrack = createDefaultTrack("next-track", "Next", "#f72585", "audio"); + useDAWStore.setState({ tracks: [sourceTrack, nextTrack], canUndo: false, canRedo: false }); + + const result = await useDAWStore.getState().convertAudioClipToMIDI("source-track", "audio-clip"); + await flushPromises(); + + const tracks = useDAWStore.getState().tracks; + expect(result).not.toBeNull(); + expect(tracks).toHaveLength(3); + expect(tracks[1].type).toBe("midi"); + expect(tracks[1].instrumentPlugin).toBeUndefined(); + expect(tracks[1].builtInInstrument).toBeUndefined(); + expect(tracks[1].midiEffects).toEqual([]); + expect(tracks[2].id).toBe("next-track"); + + const midiClip = tracks[1].midiClips[0]; + expect(midiClip.startTime).toBe(12); + expect(midiClip.duration).toBe(5); + expect(midiClip.sourceLength).toBe(5); + expect(midiClip.loopLength).toBe(5); + expect(midiClip.events).toEqual([ + { timestamp: 0.25, type: "noteOn", note: 60, velocity: 95 }, + { timestamp: 2, type: "noteOff", note: 60, velocity: 0 }, + { timestamp: 4.5, type: "noteOn", note: 67, velocity: 127 }, + { timestamp: 5, type: "noteOff", note: 67, velocity: 0 }, + ]); + expect(nativeBridge.setTrackMIDIClips).toHaveBeenCalled(); + expect(useDAWStore.getState().canUndo).toBe(true); + }); + + it("undoes and redoes the generated MIDI track as one command", async () => { + vi.mocked(nativeBridge.extractMidiFromAudio).mockResolvedValue(analysis([ + { startTime: 0, endTime: 1, midiPitch: 60, velocity: 0.8 }, + ])); + const sourceTrack = createDefaultTrack("source-track", "Audio", "#4cc9f0", "audio"); + sourceTrack.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [sourceTrack], canUndo: false, canRedo: false }); + + const result = await useDAWStore.getState().convertAudioClipToMIDI("source-track", "audio-clip"); + await flushPromises(); + const generatedTrackId = result?.trackId ?? ""; + + useDAWStore.getState().undo(); + expect(useDAWStore.getState().tracks.map((track) => track.id)).toEqual(["source-track"]); + expect(nativeBridge.removeTrack).toHaveBeenCalledWith(generatedTrackId); + + useDAWStore.getState().redo(); + await flushPromises(); + expect(useDAWStore.getState().tracks.map((track) => track.id)).toEqual(["source-track", generatedTrackId]); + expect(nativeBridge.addTrack).toHaveBeenCalledWith(generatedTrackId, "midi"); + }); + + it("creates an empty matching MIDI clip when analysis finds no notes", async () => { + const sourceTrack = createDefaultTrack("source-track", "Audio", "#4cc9f0", "audio"); + sourceTrack.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [sourceTrack], canUndo: false, canRedo: false }); + + await useDAWStore.getState().convertAudioClipToMIDI("source-track", "audio-clip"); + + const midiClip = useDAWStore.getState().tracks[1].midiClips[0]; + expect(midiClip.duration).toBe(5); + expect(midiClip.events).toEqual([]); + expect(useDAWStore.getState().toastType).toBe("info"); + }); + + it("does not create a track when the backend reports an analysis error", async () => { + vi.mocked(nativeBridge.extractMidiFromAudio).mockResolvedValue({ + ...analysis(), + error: "Polyphonic model not loaded.", + }); + const sourceTrack = createDefaultTrack("source-track", "Audio", "#4cc9f0", "audio"); + sourceTrack.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [sourceTrack], canUndo: false, canRedo: false }); + + const result = await useDAWStore.getState().convertAudioClipToMIDI("source-track", "audio-clip"); + + expect(result).toBeNull(); + expect(useDAWStore.getState().tracks).toHaveLength(1); + expect(useDAWStore.getState().toastMessage).toBe("Polyphonic model not loaded."); + }); + + it("keeps context menu and action registry wired to the store conversion action", () => { + expect(timelineSource).toContain('label: "Convert to MIDI..."'); + expect(timelineSource).toContain("convertAudioClipToMIDI(menu.trackId, menu.clipId)"); + expect(timelineSource).not.toContain("Extract MIDI from Audio..."); + expect(actionRegistrySource).toContain('name: "Convert Audio to MIDI"'); + expect(actionRegistrySource).toContain("state.convertAudioClipToMIDI(track.id, clipId)"); + }); + + it("exposes the action through the store contract", () => { + expect(typeof audioToMidiActions).toBe("function"); + expect(typeof useDAWStore.getState().convertAudioClipToMIDI).toBe("function"); + }); +}); diff --git a/frontend/src/__tests__/interactionSafetyGuards.test.ts b/frontend/src/__tests__/interactionSafetyGuards.test.ts index 0fe3556..867def0 100644 --- a/frontend/src/__tests__/interactionSafetyGuards.test.ts +++ b/frontend/src/__tests__/interactionSafetyGuards.test.ts @@ -45,6 +45,25 @@ describe("interaction safety guards", () => { expect(midiWindowSource).toContain("installModalContextMenuLeakGuard()"); }); + it("isolates modal pointer drags without blocking text selection", () => { + expect(modalGuardSource).toContain("guardModalPointerEvent"); + expect(modalGuardSource).toContain("event.stopPropagation();"); + expect(modalGuardSource).toContain("isEditableTextTarget"); + expect(modalGuardSource).toContain("document.addEventListener(\"mousemove\", stopModalPointerBubble, false)"); + expect(modalGuardSource).toContain("nativeBridge.startWindowDrag()"); + expect(modalSource).toContain("onMouseDown={guardModalPointerEvent}"); + expect(modalSource).toContain("onPointerDown={guardModalPointerEvent}"); + expect(modalSource).toContain("onMouseMove={guardModalPointerEvent}"); + expect(modalGuardSource).toContain("event.preventDefault();"); + }); + + it("keeps context submenus open while crossing into the submenu", () => { + expect(contextMenuSource).toContain("submenuCloseTimerRef"); + expect(contextMenuSource).toContain("scheduleSubmenuClose"); + expect(contextMenuSource).toContain("onMouseEnter={clearSubmenuCloseTimer}"); + expect(contextMenuSource).toContain("left-full top-0 -ml-px"); + }); + it("keeps piano roll resize and modal context-menu paths guarded", () => { expect(pianoRollSource).toContain("Math.max(1, containerRef.current.clientWidth)"); expect(pianoRollSource).toContain("Math.max(1, containerRef.current.clientHeight)"); diff --git a/frontend/src/__tests__/midiEditorTimelineRange.test.ts b/frontend/src/__tests__/midiEditorTimelineRange.test.ts new file mode 100644 index 0000000..840d158 --- /dev/null +++ b/frontend/src/__tests__/midiEditorTimelineRange.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import pianoRollSource from "../components/PianoRoll.tsx?raw"; +import timelineSource from "../components/Timeline.tsx?raw"; +import { serializeMIDIClipsForBackend } from "../utils/midiClipSerialization"; + +describe("MIDI editor timeline range", () => { + it("keeps inline piano-roll scrolling padded instead of source-end bounded", () => { + expect(pianoRollSource).toContain("EDITOR_HORIZONTAL_PADDING_BARS"); + expect(pianoRollSource).toContain("editorContentEnd"); + expect(pianoRollSource).toContain("contentDuration + editorPaddingSeconds"); + expect(pianoRollSource).toContain("loopDragEnd"); + expect(pianoRollSource).not.toContain("if (timelineScrollX > maxScrollX)"); + }); + + it("autoscrolls while dragging MIDI loop boundaries near editor edges", () => { + expect(pianoRollSource).toContain("LOOP_BOUNDARY_EDGE_SCROLL_ZONE_PX"); + expect(pianoRollSource).toContain("LOOP_BOUNDARY_EDGE_SCROLL_MAX_STEP_PX"); + expect(pianoRollSource).toContain("currentSourceTime"); + expect(pianoRollSource).toContain("projectXToMidiSourceTime"); + }); + + it("counts MIDI clips when sizing the main timeline scroll range", () => { + expect(timelineSource).toContain("track.midiClips.forEach"); + expect(timelineSource).toContain("const clipEnd = clip.startTime + clip.duration"); + }); + + it("preserves overlapping MIDI clips as separate scheduled clips", () => { + const clips = [ + { + id: "clip-a", + startTime: 10, + duration: 4, + sourceLength: 4, + loopLength: 4, + events: [ + { type: "noteOn", timestamp: 0, note: 60, velocity: 100 }, + { type: "noteOff", timestamp: 2, note: 60, velocity: 0 }, + ], + ccEvents: [], + }, + { + id: "clip-b", + startTime: 11, + duration: 4, + sourceLength: 4, + loopLength: 4, + events: [ + { type: "noteOn", timestamp: 0, note: 64, velocity: 100 }, + { type: "noteOff", timestamp: 2, note: 64, velocity: 0 }, + ], + ccEvents: [], + }, + ] as any; + + const serialized = serializeMIDIClipsForBackend(clips); + + expect(serialized).toHaveLength(2); + expect(serialized.map((clip) => clip.startTime)).toEqual([10, 11]); + expect(serialized[0].events.map((event) => event.timestamp)).toEqual([0, 2]); + expect(serialized[1].events.map((event) => event.timestamp)).toEqual([0, 2]); + }); +}); diff --git a/frontend/src/__tests__/renderInPlaceAlignment.test.ts b/frontend/src/__tests__/renderInPlaceAlignment.test.ts new file mode 100644 index 0000000..7824a51 --- /dev/null +++ b/frontend/src/__tests__/renderInPlaceAlignment.test.ts @@ -0,0 +1,159 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { nativeBridge } from "../services/NativeBridge"; +import { commandManager } from "../store/commands"; +import { createDefaultTrack, type AudioClip, type MIDIClip, useDAWStore } from "../store/useDAWStore"; + +const initialState = useDAWStore.getState(); + +function flushPromises() { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function audioClip(overrides: Partial = {}): AudioClip { + return { + id: "audio-clip", + filePath: "C:/audio/source.wav", + name: "Lead Vocal", + startTime: 2, + duration: 6, + offset: 0, + color: "#f72585", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + sampleRate: 48000, + sourceLength: 6, + ...overrides, + }; +} + +function midiClip(overrides: Partial = {}): MIDIClip { + return { + id: "midi-clip", + name: "One Bar Late", + startTime: 4, + duration: 8, + offset: 0, + sourceStart: 0, + sourceLength: 8, + loopEnabled: false, + loopOffset: 0, + loopLength: 8, + events: [ + { timestamp: 2, type: "noteOn", note: 60, velocity: 96 }, + { timestamp: 3, type: "noteOff", note: 60, velocity: 0 }, + ], + ccEvents: [], + color: "#38bdf8", + ...overrides, + }; +} + +describe("render in place alignment", () => { + beforeEach(() => { + commandManager.clear(); + useDAWStore.setState(initialState); + vi.spyOn(nativeBridge, "showRenderSaveDialog").mockResolvedValue("C:/renders/midi-render.wav"); + vi.spyOn(nativeBridge, "renderProject").mockResolvedValue(true); + vi.spyOn(nativeBridge, "importMediaFile").mockResolvedValue({ + filePath: "C:/renders/midi-render.wav", + duration: 9.25, + sampleRate: 48000, + numChannels: 2, + format: "wav", + }); + vi.spyOn(nativeBridge, "addTrack").mockResolvedValue("rendered-track"); + vi.spyOn(nativeBridge, "reorderTrack").mockResolvedValue(true); + vi.spyOn(nativeBridge, "addPlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "removePlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "removeTrack").mockResolvedValue(true); + vi.spyOn(nativeBridge, "refreshWaveformPeaks").mockResolvedValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + commandManager.clear(); + useDAWStore.setState(initialState); + }); + + it("renders the requested MIDI timeline range without adding a visible tail", async () => { + const sourceTrack = createDefaultTrack("track-source", "Instrument", "#38bdf8", "instrument"); + sourceTrack.midiClips = [midiClip()]; + const syncClipsWithBackend = vi.fn().mockResolvedValue(undefined); + const syncMIDITrackToBackend = vi.fn().mockResolvedValue(undefined); + useDAWStore.setState({ + tracks: [sourceTrack], + syncClipsWithBackend, + syncMIDITrackToBackend, + canUndo: false, + canRedo: false, + }); + + await useDAWStore.getState().renderClipInPlace("midi-clip"); + await flushPromises(); + + expect(nativeBridge.renderProject).toHaveBeenCalledWith(expect.objectContaining({ + source: "stem:track-source", + startTime: 4, + endTime: 12, + addTail: false, + tailLength: 0, + })); + + const tracks = useDAWStore.getState().tracks; + const renderedClip = tracks[1].clips[0]; + expect(renderedClip).toMatchObject({ + filePath: "C:/renders/midi-render.wav", + startTime: 4, + duration: 8, + sourceLength: 9.25, + sampleRate: 48000, + }); + expect(tracks[0].midiClips[0].muted).toBe(true); + expect(nativeBridge.refreshWaveformPeaks).toHaveBeenCalledWith("C:/renders/midi-render.wav"); + expect(useDAWStore.getState().canUndo).toBe(true); + }); + + it("resyncs audio clip render-in-place from muted frontend state instead of directly double-registering clips", async () => { + vi.mocked(nativeBridge.showRenderSaveDialog).mockResolvedValue("C:/renders/audio-render.wav"); + vi.mocked(nativeBridge.importMediaFile).mockResolvedValue({ + filePath: "C:/renders/audio-render.wav", + duration: 6.5, + sampleRate: 48000, + numChannels: 2, + format: "wav", + }); + + const sourceTrack = createDefaultTrack("track-source", "Vocals", "#f72585", "audio"); + sourceTrack.clips = [audioClip()]; + const syncedSnapshots: Array> = []; + const syncClipsWithBackend = vi.fn().mockImplementation(async () => { + syncedSnapshots.push(useDAWStore.getState().tracks.flatMap((track) => + track.clips + .filter((clip) => clip.filePath && !clip.muted) + .map((clip) => ({ trackId: track.id, clipId: clip.id, filePath: clip.filePath })), + )); + }); + useDAWStore.setState({ + tracks: [sourceTrack], + syncClipsWithBackend, + canUndo: false, + canRedo: false, + }); + + await useDAWStore.getState().renderClipInPlace("audio-clip"); + await flushPromises(); + + expect(nativeBridge.addPlaybackClip).not.toHaveBeenCalled(); + expect(syncClipsWithBackend).toHaveBeenCalledTimes(2); + expect(syncedSnapshots[0]).toEqual([ + { trackId: "track-source", clipId: "audio-clip", filePath: "C:/audio/source.wav" }, + ]); + const finalSync = syncedSnapshots[syncedSnapshots.length - 1] ?? []; + expect(finalSync).toHaveLength(1); + expect(finalSync[0]).toMatchObject({ + filePath: "C:/renders/audio-render.wav", + }); + expect(useDAWStore.getState().tracks[0].clips[0].muted).toBe(true); + }); +}); diff --git a/frontend/src/__tests__/timelineDragAxisLock.test.ts b/frontend/src/__tests__/timelineDragAxisLock.test.ts new file mode 100644 index 0000000..4c47020 --- /dev/null +++ b/frontend/src/__tests__/timelineDragAxisLock.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + getTimelineAxisLockedDeltas, + resolveTimelineDragAxisLock, +} from "../utils/timelineDragAxisLock"; + +describe("timeline drag axis lock", () => { + it("does not lock or move before the drag threshold", () => { + expect(resolveTimelineDragAxisLock(true, null, 3, 5)).toBeNull(); + expect(getTimelineAxisLockedDeltas(true, null, 3, 5)).toEqual({ + axisLock: null, + deltaX: 0, + deltaY: 0, + }); + }); + + it("locks horizontally when horizontal movement dominates", () => { + expect(resolveTimelineDragAxisLock(true, null, 12, 4)).toBe("x"); + expect(getTimelineAxisLockedDeltas(true, null, 12, 4)).toEqual({ + axisLock: "x", + deltaX: 12, + deltaY: 0, + }); + }); + + it("locks vertically when vertical movement dominates", () => { + expect(resolveTimelineDragAxisLock(true, null, 4, 12)).toBe("y"); + expect(getTimelineAxisLockedDeltas(true, null, 4, 12)).toEqual({ + axisLock: "y", + deltaX: 0, + deltaY: 12, + }); + }); + + it("chooses horizontal lock on ties", () => { + expect(resolveTimelineDragAxisLock(true, null, 8, -8)).toBe("x"); + }); + + it("keeps an existing lock for the rest of the drag", () => { + expect(resolveTimelineDragAxisLock(true, "y", 30, 1)).toBe("y"); + expect(getTimelineAxisLockedDeltas(true, "y", 30, 1)).toEqual({ + axisLock: "y", + deltaX: 0, + deltaY: 1, + }); + }); + + it("does not constrain deltas when Shift axis lock was not requested", () => { + expect(getTimelineAxisLockedDeltas(false, null, 4, 12)).toEqual({ + axisLock: null, + deltaX: 4, + deltaY: 12, + }); + }); +}); diff --git a/frontend/src/__tests__/timelineTrackDrop.test.ts b/frontend/src/__tests__/timelineTrackDrop.test.ts new file mode 100644 index 0000000..ae8c857 --- /dev/null +++ b/frontend/src/__tests__/timelineTrackDrop.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import timelineSource from "../components/Timeline.tsx?raw"; +import trackActionsSource from "../store/actions/tracks.ts?raw"; +import storeSource from "../store/useDAWStore.ts?raw"; + +describe("timeline track drop regression guards", () => { + it("creates ghost-drop tracks with one explicit backend-created id path", () => { + expect(timelineSource).toContain("finalizingTimelineClipGestureRef"); + expect(timelineSource).toContain("createTimelineGeneratedTrack"); + expect(timelineSource).toContain("nativeBridge.addTrack(trackId, newTrackType)"); + expect(timelineSource).toContain("backendAlreadyCreated: true"); + expect(timelineSource).not.toContain("nativeBridge.addTrack(undefined, newTrackType)"); + }); + + it("keeps audio clips on new audio tracks and midi clips on new midi tracks", () => { + expect(timelineSource).toContain('function getNewTrackTypeForTimelineClip(isMidi: boolean): "audio" | "midi"'); + expect(timelineSource).toContain('return isMidi ? "midi" : "audio";'); + }); + + it("lets undoable addTrack skip duplicate ids and backend re-adds when already created", () => { + expect(storeSource).toContain("backendAlreadyCreated?: boolean"); + expect(trackActionsSource).toContain("Ignoring duplicate addTrack"); + expect(trackActionsSource).toContain("const includeAddTrack = !(options.backendAlreadyCreated && !hasExecutedOnce)"); + expect(trackActionsSource).toContain("syncTrackCoreToBackend(fullTrack, { includeAddTrack })"); + }); +}); diff --git a/frontend/src/__tests__/transportAutoStop.test.ts b/frontend/src/__tests__/transportAutoStop.test.ts new file mode 100644 index 0000000..e0dd7c6 --- /dev/null +++ b/frontend/src/__tests__/transportAutoStop.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from "vitest"; +import appSource from "../App.tsx?raw"; +import { + getPlaybackContentBounds, + shouldAutoStopPlayback, + type TrackWithArrangementClips, +} from "../utils/transportAutoStop"; + +function arrangementClip(startTime: number, duration: number, extras: Record = {}) { + return { id: `${startTime}-${duration}`, startTime, duration, ...extras } as any; +} + +function arrangementTrack( + clips: any[] = [], + midiClips: any[] = [], + extras: Record = {}, +): TrackWithArrangementClips { + return { clips, midiClips, ...extras } as unknown as TrackWithArrangementClips; +} + +function transport(overrides: Record = {}) { + return { + isPlaying: true, + isRecording: false, + currentTime: 0, + loopEnabled: false, + loopStart: 0, + loopEnd: 0, + ...overrides, + }; +} + +describe("transport auto-stop", () => { + it("auto-stops an empty project when the metronome is off", () => { + const decision = shouldAutoStopPlayback({ + tracks: [], + transport: transport(), + metronomeEnabled: false, + nextTime: 0.016, + }); + + expect(decision.shouldStop).toBe(true); + expect(decision.reason).toBe("empty"); + }); + + it("keeps playing an empty project when the metronome is on", () => { + const decision = shouldAutoStopPlayback({ + tracks: [], + transport: transport(), + metronomeEnabled: true, + nextTime: 0.016, + }); + + expect(decision.shouldStop).toBe(false); + }); + + it("keeps recording even when no arrangement clips exist", () => { + const decision = shouldAutoStopPlayback({ + tracks: [], + transport: transport({ isRecording: true }), + metronomeEnabled: false, + nextTime: 0.016, + }); + + expect(decision.shouldStop).toBe(false); + }); + + it("counts both audio clips and MIDI clips when finding the latest playable end", () => { + const bounds = getPlaybackContentBounds([ + arrangementTrack([arrangementClip(2, 3)], [arrangementClip(8, 4)]), + ]); + + expect(bounds.hasContent).toBe(true); + expect(bounds.latestEndTime).toBe(12); + }); + + it("counts muted clips and clips on muted or unsoloed tracks as timeline content", () => { + const decision = shouldAutoStopPlayback({ + tracks: [ + arrangementTrack( + [arrangementClip(0, 10, { muted: true })], + [], + { muted: true, solo: false }, + ), + ], + transport: transport({ currentTime: 2 }), + metronomeEnabled: false, + nextTime: 2.016, + }); + + expect(decision.shouldStop).toBe(false); + expect(decision.bounds.latestEndTime).toBe(10); + }); + + it("ignores zero-duration and invalid clips", () => { + const bounds = getPlaybackContentBounds([ + arrangementTrack([ + arrangementClip(0, 0), + arrangementClip(1, -1), + arrangementClip(Number.NaN, 4), + ]), + ]); + + expect(bounds.hasContent).toBe(false); + expect(bounds.latestEndTime).toBe(0); + }); + + it("auto-stops when non-looping playback reaches the latest clip end", () => { + const decision = shouldAutoStopPlayback({ + tracks: [arrangementTrack([arrangementClip(1, 4)])], + transport: transport({ currentTime: 4.99 }), + metronomeEnabled: false, + nextTime: 5.01, + }); + + expect(decision.shouldStop).toBe(true); + expect(decision.reason).toBe("end-of-content"); + expect(decision.stopTime).toBe(5); + }); + + it("auto-stops a loop region that contains no overlapping clips", () => { + const decision = shouldAutoStopPlayback({ + tracks: [arrangementTrack([arrangementClip(1, 4)])], + transport: transport({ + currentTime: 20, + loopEnabled: true, + loopStart: 20, + loopEnd: 24, + }), + metronomeEnabled: false, + nextTime: 20.016, + }); + + expect(decision.shouldStop).toBe(true); + expect(decision.reason).toBe("silent-loop"); + }); + + it("keeps looping when the loop region overlaps any clip", () => { + const decision = shouldAutoStopPlayback({ + tracks: [arrangementTrack([arrangementClip(22, 1)])], + transport: transport({ + currentTime: 23.99, + loopEnabled: true, + loopStart: 20, + loopEnd: 24, + }), + metronomeEnabled: false, + nextTime: 24.01, + }); + + expect(decision.shouldStop).toBe(false); + expect(decision.bounds.loopHasContent).toBe(true); + }); + + it("checks auto-stop before loop wrapping in the app playback loop", () => { + const autoStopIndex = appSource.indexOf("const autoStopDecision = shouldAutoStopPlayback"); + const loopWrapIndex = appSource.indexOf("// Loop: wrap back to loopStart"); + + expect(autoStopIndex).toBeGreaterThanOrEqual(0); + expect(loopWrapIndex).toBeGreaterThan(autoStopIndex); + expect(appSource).toContain("let autoStopInFlight = false"); + expect(appSource).toContain("void currentState.stop()"); + }); +}); diff --git a/frontend/src/__tests__/transportRecording.test.ts b/frontend/src/__tests__/transportRecording.test.ts new file mode 100644 index 0000000..303b773 --- /dev/null +++ b/frontend/src/__tests__/transportRecording.test.ts @@ -0,0 +1,224 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { nativeBridge } from "../services/NativeBridge"; +import { createDefaultTrack, type Track, useDAWStore } from "../store/useDAWStore"; +import transportBarSource from "../components/TransportBar.tsx?raw"; + +const initialState = useDAWStore.getState(); + +function flushPromises() { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function armedInstrumentTrack(overrides: Partial = {}): Track { + return { + ...createDefaultTrack("track-midi", "Instrument", "#38bdf8", "instrument"), + armed: true, + inputType: "midi" as const, + midiInputDevice: "Keyboard", + midiChannel: 0, + ...overrides, + }; +} + +describe("transport recording", () => { + beforeEach(() => { + useDAWStore.setState(initialState); + vi.spyOn(nativeBridge, "addTrack").mockResolvedValue("track-midi"); + vi.spyOn(nativeBridge, "setTrackType").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTrackRecordArm").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTrackInputMonitoring").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTrackInputChannels").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTrackMIDIInput").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTransportPosition").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTransportPlaying").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setTransportRecording").mockResolvedValue(true); + vi.spyOn(nativeBridge, "setPunchRange").mockResolvedValue(true); + vi.spyOn(nativeBridge, "getLastCompletedClips").mockResolvedValue([]); + vi.spyOn(nativeBridge, "getLastCompletedMIDIClips").mockResolvedValue([]); + vi.spyOn(nativeBridge, "addPlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "refreshWaveformPeaks").mockResolvedValue(true); + vi.spyOn(nativeBridge, "getAudioDebugSnapshot").mockResolvedValue({} as any); + vi.spyOn(nativeBridge, "getMIDIInputDevices").mockResolvedValue(["Keyboard"]); + vi.spyOn(nativeBridge, "getOpenMIDIDevices").mockResolvedValue(["Keyboard"]); + vi.spyOn(nativeBridge, "openMIDIDevice").mockResolvedValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + useDAWStore.setState(initialState); + }); + + it("lets the bottom transport record button toggle recording", () => { + expect(transportBarSource).toContain("await toggleRecord();"); + expect(transportBarSource).not.toContain("if (isRecording) return;"); + expect(transportBarSource).not.toContain("await record();"); + }); + + it("punches in at the playhead captured when record is pressed", async () => { + const calls: string[] = []; + vi.mocked(nativeBridge.setTransportRecording).mockImplementation(async (recording) => { + calls.push(`record:${recording}`); + return true; + }); + vi.mocked(nativeBridge.getMIDIInputDevices).mockImplementation(async () => { + calls.push("get-midi-devices"); + return ["Keyboard"]; + }); + + const syncClipsWithBackend = vi.fn().mockResolvedValue(undefined); + useDAWStore.setState({ + tracks: [armedInstrumentTrack()], + transport: { + ...useDAWStore.getState().transport, + isPlaying: true, + isPaused: false, + isRecording: false, + currentTime: 12.345, + }, + recordingClips: [], + recordingMIDIPreviews: {}, + syncClipsWithBackend, + }); + + await useDAWStore.getState().record(); + await flushPromises(); + + expect(syncClipsWithBackend).not.toHaveBeenCalled(); + expect(nativeBridge.setTransportPlaying).not.toHaveBeenCalled(); + expect(nativeBridge.setTransportRecording).toHaveBeenCalledWith(true); + expect(useDAWStore.getState().recordingClips).toEqual([ + { trackId: "track-midi", startTime: 12.345 }, + ]); + expect(calls.indexOf("record:true")).toBeGreaterThanOrEqual(0); + expect(calls.indexOf("get-midi-devices")).toBeGreaterThan(calls.indexOf("record:true")); + }); + + it("toggles record to stop when a recording session is active", async () => { + const stop = vi.fn().mockResolvedValue(undefined); + useDAWStore.setState({ + stop, + recordSession: { id: "record-session", startTime: 3, trackIds: ["track-midi"] }, + transport: { + ...useDAWStore.getState().transport, + isRecording: true, + }, + }); + + await useDAWStore.getState().toggleRecord(); + + expect(stop).toHaveBeenCalled(); + }); + + it("selects newly completed recorded clips after stop", async () => { + const audioTrack = { + ...createDefaultTrack("track-audio", "Audio", "#f97316", "audio"), + armed: true, + }; + const midiTrack = armedInstrumentTrack(); + + vi.mocked(nativeBridge.getLastCompletedClips).mockResolvedValue([ + { + trackId: audioTrack.id, + filePath: "C:/recordings/take.wav", + startTime: 2, + duration: 3, + }, + ]); + vi.mocked(nativeBridge.getLastCompletedMIDIClips).mockResolvedValue([ + { + trackId: midiTrack.id, + startTime: 2, + duration: 3, + events: [ + { timestamp: 0, type: "noteOn", note: 60, velocity: 96, channel: 1 }, + { timestamp: 1, type: "noteOff", note: 60, velocity: 0, channel: 1 }, + ], + }, + ]); + + useDAWStore.setState({ + tracks: [audioTrack, midiTrack], + selectedClipIds: ["old-clip"], + selectedClipId: "old-clip", + selectedTrackId: audioTrack.id, + selectedTrackIds: [audioTrack.id], + lastSelectedTrackId: audioTrack.id, + transport: { + ...useDAWStore.getState().transport, + isPlaying: true, + isPaused: false, + isRecording: true, + currentTime: 5, + }, + playStartPosition: 2, + syncMIDITrackToBackend: vi.fn().mockResolvedValue(undefined), + }); + + await useDAWStore.getState().stop(); + + const state = useDAWStore.getState(); + const recordedAudioClip = state.tracks.find((track) => track.id === audioTrack.id)?.clips[0]; + const recordedMIDIClip = state.tracks.find((track) => track.id === midiTrack.id)?.midiClips[0]; + + expect(recordedAudioClip).toBeDefined(); + expect(recordedMIDIClip).toBeDefined(); + expect(state.selectedClipIds).toEqual([recordedAudioClip!.id, recordedMIDIClip!.id]); + expect(state.selectedClipId).toBe(recordedMIDIClip!.id); + expect(state.selectedTrackId).toBeNull(); + expect(state.selectedTrackIds).toEqual([]); + expect(state.lastSelectedTrackId).toBeNull(); + }); + + it("finalizes completed clips when transport sync cleared isRecording but a record session remains", async () => { + const audioTrack = { + ...createDefaultTrack("track-audio", "Audio", "#f97316", "audio"), + armed: false, + }; + vi.mocked(nativeBridge.getLastCompletedClips).mockResolvedValue([ + { + trackId: audioTrack.id, + filePath: "C:/recordings/take.wav", + startTime: 2, + duration: 3, + }, + ]); + + useDAWStore.setState({ + tracks: [audioTrack], + recordSession: { id: "record-session", startTime: 2, trackIds: [audioTrack.id] }, + transport: { + ...useDAWStore.getState().transport, + isPlaying: false, + isPaused: false, + isRecording: false, + currentTime: 5, + }, + playStartPosition: 2, + }); + + await useDAWStore.getState().stop(); + + const recordedAudioClip = useDAWStore.getState().tracks[0].clips[0]; + expect(nativeBridge.getLastCompletedClips).toHaveBeenCalled(); + expect(recordedAudioClip).toMatchObject({ + filePath: "C:/recordings/take.wav", + startTime: 2, + duration: 3, + }); + }); + + it("prepares MIDI input routing when an instrument track is armed", async () => { + useDAWStore.setState({ + tracks: [armedInstrumentTrack({ armed: false, midiInputDevice: "" })], + }); + + await useDAWStore.getState().toggleTrackArmed("track-midi"); + + expect(nativeBridge.addTrack).toHaveBeenCalledWith("track-midi", "instrument"); + expect(nativeBridge.setTrackType).toHaveBeenCalledWith("track-midi", "instrument"); + expect(nativeBridge.setTrackRecordArm).toHaveBeenCalledWith("track-midi", true); + expect(nativeBridge.getMIDIInputDevices).toHaveBeenCalled(); + expect(nativeBridge.openMIDIDevice).toHaveBeenCalledWith("Keyboard"); + expect(nativeBridge.setTrackMIDIInput).toHaveBeenCalledWith("track-midi", "", 0); + }); +}); diff --git a/frontend/src/components/AIClipGenerationModal.tsx b/frontend/src/components/AIClipGenerationModal.tsx new file mode 100644 index 0000000..6949878 --- /dev/null +++ b/frontend/src/components/AIClipGenerationModal.tsx @@ -0,0 +1,845 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { AlertTriangle, CheckCircle2, ChevronDown, Music2, Settings2, WandSparkles } from "lucide-react"; +import { useShallow } from "zustand/shallow"; +import { + ACE_STEP_MODEL_ID, + AI_WORKFLOW_SECTION_LABELS, + STABLE_AUDIO_3_MODEL_ID, + type AIWorkflowParam, + type AIWorkflowSection, + getAIModelsForWorkflow, + getAIWorkflow, + getAiMusicModel, + getClipInpaintRange, + mergeWorkflowParams, + normalizeWorkflowParams, + resolveAiMusicModelId, +} from "../data/aiWorkflows"; +import { nativeBridge, type AIGenerationProgress } from "../services/NativeBridge"; +import { type AudioClip, useDAWStore } from "../store/useDAWStore"; +import { + Button, + Checkbox, + Input, + Modal, + ModalContent, + ModalFooter, + ModalHeader, + Select, + Textarea, +} from "./ui"; +import { NumericWorkflowParamField } from "./AIWorkflowParamField"; + +const SECTION_ORDER: AIWorkflowSection[] = [ + "prompt", + "source", + "sampling", + "generation", + "advanced", +]; + +const ADVANCED_SECTIONS = new Set([ + "sampling", + "generation", + "advanced", +]); + +const RANGE_PARAM_KEYS = new Set([ + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", +]); + +const STABLE_SOURCE_WORKFLOW_IDS = new Set([ + "variation", + "inpaint-selection", + "continue-clip", +]); + +export function buildAIClipGenerationRequestParams({ + params, + modelId, + sourceTrack, + sourceClip, + workflowRange, + extensionDuration, + transportTempo, + timeSignature, +}: { + params: Record; + modelId: string; + sourceTrack: { id: string; name: string }; + sourceClip: AudioClip; + workflowRange: { start: number; end: number } | null; + extensionDuration: number; + transportTempo: number; + timeSignature: { numerator: number; denominator: number }; +}) { + const requestParams: Record = { ...params }; + if (modelId === ACE_STEP_MODEL_ID) { + requestParams.bpm = Math.round(Number.isFinite(transportTempo) ? transportTempo : 120); + requestParams.timesignature = `${timeSignature.numerator}/${timeSignature.denominator}`; + } + + return { + ...requestParams, + source: { + filePath: sourceClip.filePath, + clipOffset: sourceClip.offset || 0, + clipDuration: sourceClip.duration, + sourceTrackId: sourceTrack.id, + sourceTrackName: sourceTrack.name, + sourceClipId: sourceClip.id, + sourceClipName: sourceClip.name, + sourceClipStartTime: sourceClip.startTime, + inpaintRange: workflowRange, + extensionDuration, + }, + }; +} + +function formatDuration(seconds: number) { + const safeSeconds = Math.max(0, seconds || 0); + const minutes = Math.floor(safeSeconds / 60); + const wholeSeconds = Math.floor(safeSeconds % 60); + return `${minutes}:${wholeSeconds.toString().padStart(2, "0")}`; +} + +function formatRange(range: { start: number; end: number } | null) { + if (!range) return "No overlapping time selection"; + return `${range.start.toFixed(2)}s - ${range.end.toFixed(2)}s`; +} + +function formatPhaseLabel(phase?: string) { + if (!phase) return "Preparing"; + return phase + .split("_") + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function formatElapsedLabel(elapsedMs?: number) { + if (!elapsedMs || elapsedMs <= 0) return ""; + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s elapsed` : `${seconds}s elapsed`; +} + +function formatRuntimeProfileLabel(profile?: string) { + switch (profile) { + case "ace-diffusers": + return "ACE-Step Diffusers"; + default: + return profile ? formatPhaseLabel(profile) : ""; + } +} + +function formatSessionModeLabel(sessionMode?: string) { + switch (sessionMode) { + case "persistent": + return "Persistent session"; + case "oneshot-fallback": + return "One-shot fallback"; + case "oneshot": + return "One-shot"; + default: + return sessionMode ? formatPhaseLabel(sessionMode) : ""; + } +} + +function progressWidth(progress?: number) { + return `${Math.max(4, Math.round((progress ?? 0) * 100))}%`; +} + +function shouldShowParam(workflowId: string, param: AIWorkflowParam) { + if (workflowId === "variation") { + return ![ + "extension_duration", + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", + ].includes(param.key); + } + + if (workflowId === "inpaint-selection") { + return !["extension_duration"].includes(param.key); + } + + if (workflowId === "continue-clip") { + return ![ + "duration", + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", + ].includes(param.key); + } + + return true; +} + +export default function AIClipGenerationModal() { + const { + showAIClipGeneration, + aiClipGenerationTrackId, + aiClipGenerationClipId, + aiClipGenerationWorkflowId, + aiClipGenerationModelId, + aiClipGenerationParams, + aiClipGenerationRange, + aiClipGenerationError, + tracks, + transportTempo, + timeSignature, + aiToolsStatus, + closeAIClipGeneration, + setAIClipGenerationModel, + setAIClipGenerationParams, + setAIClipGenerationRange, + setAIClipGenerationError, + addGeneratedSourceAudioClip, + openAiToolsSetup, + } = useDAWStore( + useShallow((state) => ({ + showAIClipGeneration: state.showAIClipGeneration, + aiClipGenerationTrackId: state.aiClipGenerationTrackId, + aiClipGenerationClipId: state.aiClipGenerationClipId, + aiClipGenerationWorkflowId: state.aiClipGenerationWorkflowId, + aiClipGenerationModelId: state.aiClipGenerationModelId, + aiClipGenerationParams: state.aiClipGenerationParams, + aiClipGenerationRange: state.aiClipGenerationRange, + aiClipGenerationError: state.aiClipGenerationError, + tracks: state.tracks, + transportTempo: state.transport.tempo, + timeSignature: state.timeSignature, + aiToolsStatus: state.aiToolsStatus, + closeAIClipGeneration: state.closeAIClipGeneration, + setAIClipGenerationModel: state.setAIClipGenerationModel, + setAIClipGenerationParams: state.setAIClipGenerationParams, + setAIClipGenerationRange: state.setAIClipGenerationRange, + setAIClipGenerationError: state.setAIClipGenerationError, + addGeneratedSourceAudioClip: state.addGeneratedSourceAudioClip, + openAiToolsSetup: state.openAiToolsSetup, + })), + ); + + const [isGenerating, setIsGenerating] = useState(false); + const [advancedOpen, setAdvancedOpen] = useState(false); + const [detailsOpen, setDetailsOpen] = useState(false); + const [progress, setProgress] = useState({ state: "idle", progress: 0 }); + const pollRef = useRef | null>(null); + const completedRef = useRef(false); + const contentRef = useRef(null); + const wasGeneratingRef = useRef(false); + + const sourceTrack = useMemo( + () => tracks.find((track) => track.id === aiClipGenerationTrackId), + [aiClipGenerationTrackId, tracks], + ); + const sourceClip = useMemo( + () => sourceTrack?.clips.find((clip) => clip.id === aiClipGenerationClipId), + [aiClipGenerationClipId, sourceTrack], + ); + const workflow = getAIWorkflow( + aiClipGenerationWorkflowId, + aiClipGenerationModelId, + "clip-context", + ); + const model = getAiMusicModel(aiClipGenerationModelId); + const supportedModels = getAIModelsForWorkflow(workflow.id); + const params = useMemo( + () => mergeWorkflowParams(workflow.id, aiClipGenerationParams, aiClipGenerationModelId), + [aiClipGenerationModelId, aiClipGenerationParams, workflow.id], + ); + const selectedModelStatus = aiToolsStatus.musicModels?.[aiClipGenerationModelId]; + const isModelReady = Boolean( + selectedModelStatus?.ready + ?? ( + aiClipGenerationModelId === ACE_STEP_MODEL_ID + ? ( + aiToolsStatus.features?.audioGeneration?.ready + ?? ( + aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + ) + ) + : false + ), + ); + const modelBlockedMessage = + selectedModelStatus?.message + || selectedModelStatus?.blockReason + || aiToolsStatus.features?.audioGeneration?.message + || aiToolsStatus.musicGenerationStatusMessage + || aiToolsStatus.message + || `${model.label} is not set up yet.`; + + const currentInpaintRange = sourceClip + ? getClipInpaintRange(useDAWStore.getState().timeSelection, sourceClip) + : null; + + useEffect(() => { + if (!showAIClipGeneration || !sourceClip) return; + + const nextRange = workflow.id === "inpaint-selection" ? currentInpaintRange : null; + const sameRange = + (!nextRange && !aiClipGenerationRange) + || ( + nextRange + && aiClipGenerationRange + && Math.abs(nextRange.start - aiClipGenerationRange.start) < 0.001 + && Math.abs(nextRange.end - aiClipGenerationRange.end) < 0.001 + ); + if (!sameRange) { + setAIClipGenerationRange(nextRange); + } + + const nextParams = { + ...params, + duration: sourceClip.duration, + ...(nextRange + ? { + repainting_start: nextRange.start, + repainting_end: nextRange.end, + inpaint_start: nextRange.start, + inpaint_end: nextRange.end, + } + : {}), + }; + const normalized = normalizeWorkflowParams(workflow.id, nextParams, aiClipGenerationModelId); + if (JSON.stringify(normalized) !== JSON.stringify(params)) { + setAIClipGenerationParams(normalized); + } + }, [ + aiClipGenerationModelId, + aiClipGenerationRange, + currentInpaintRange, + params, + setAIClipGenerationParams, + setAIClipGenerationRange, + showAIClipGeneration, + sourceClip, + workflow.id, + ]); + + const stopPolling = () => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + }; + + useEffect(() => stopPolling, []); + + const paramsBySection = useMemo(() => { + return SECTION_ORDER.map((section) => ({ + section, + params: workflow.params.filter((param) => ( + param.section === section && shouldShowParam(workflow.id, param) + )), + })).filter((group) => group.params.length > 0); + }, [workflow.id, workflow.params]); + + const primaryParamGroups = paramsBySection.filter((group) => !ADVANCED_SECTIONS.has(group.section)); + const advancedParamGroups = paramsBySection.filter((group) => ADVANCED_SECTIONS.has(group.section)); + const advancedControlCount = advancedParamGroups.reduce((count, group) => count + group.params.length, 0); + const inpaintSelectionMissing = workflow.id === "inpaint-selection" && !aiClipGenerationRange; + const stableSourcePromptMissing = + aiClipGenerationModelId === STABLE_AUDIO_3_MODEL_ID + && STABLE_SOURCE_WORKFLOW_IDS.has(workflow.id) + && String(params.prompt ?? "").trim().length === 0; + const progressChips = [ + progress.backend ? progress.backend.toUpperCase() : "", + progress.runMode ? progress.runMode.toUpperCase() : "", + formatSessionModeLabel(progress.sessionMode), + formatRuntimeProfileLabel(progress.runtimeProfile), + ].filter(Boolean); + const hasProgressDetails = Boolean( + progress.statusNote + || progress.sourcePatternWarning + || progressChips.length + || progress.lastStderrLine + || progress.lastStdoutLine, + ); + + const handleParamChange = (key: string, value: unknown) => { + setAIClipGenerationParams( + normalizeWorkflowParams(workflow.id, { + ...params, + [key]: value, + }, aiClipGenerationModelId), + ); + }; + + const scrollStatusIntoView = () => { + contentRef.current?.scrollTo({ top: 0, behavior: "smooth" }); + }; + + useEffect(() => { + if (isGenerating && !wasGeneratingRef.current) { + scrollStatusIntoView(); + } + wasGeneratingRef.current = isGenerating; + }, [isGenerating]); + + const handleCancel = async () => { + if (isGenerating) { + completedRef.current = true; + stopPolling(); + await nativeBridge.cancelAIGeneration(); + setIsGenerating(false); + setProgress({ state: "idle", progress: 0 }); + return; + } + closeAIClipGeneration(); + }; + + const handleGenerate = async () => { + if (!sourceTrack || !sourceClip || !aiClipGenerationWorkflowId) { + setAIClipGenerationError("Source clip is no longer available."); + return; + } + if (!isModelReady) { + openAiToolsSetup("audioGeneration"); + return; + } + if (stableSourcePromptMissing) { + setAIClipGenerationError("Stable Audio source workflows need a direction prompt."); + return; + } + if (inpaintSelectionMissing) { + setAIClipGenerationError("Create a time selection that overlaps this clip before inpainting."); + return; + } + + completedRef.current = false; + setIsGenerating(true); + setProgress({ state: "loading", progress: 0.01, phase: "starting" }); + setAIClipGenerationError(""); + scrollStatusIntoView(); + + const extensionDuration = Number(params.extension_duration ?? 20); + const requestParams = buildAIClipGenerationRequestParams({ + params, + modelId: aiClipGenerationModelId, + sourceTrack, + sourceClip, + workflowRange: aiClipGenerationRange, + extensionDuration, + transportTempo, + timeSignature, + }); + + try { + const result = await nativeBridge.startAIGeneration( + sourceTrack.id, + aiClipGenerationModelId, + workflow.id, + requestParams, + ); + + if (!result.started) { + setIsGenerating(false); + setProgress({ state: "error", progress: 0, error: result.error }); + setAIClipGenerationError(result.error || "Failed to start AI generation."); + return; + } + + let idleCount = 0; + pollRef.current = setInterval(async () => { + if (completedRef.current) return; + const nextProgress = await nativeBridge.getAIGenerationProgress(); + if (completedRef.current) return; + setProgress(nextProgress); + + if (nextProgress.state === "idle") { + idleCount += 1; + if (idleCount >= 10) { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError("Generation did not start. Open AI Tools Setup and check the selected model."); + } + return; + } + idleCount = 0; + + if (nextProgress.state === "done") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + if (!nextProgress.outputFile) { + setAIClipGenerationError("Generation finished without producing an audio file."); + return; + } + await addGeneratedSourceAudioClip({ + sourceTrackId: sourceTrack.id, + sourceClipId: sourceClip.id, + workflowId: workflow.id, + filePath: nextProgress.outputFile, + extensionDuration, + }); + setTimeout(() => closeAIClipGeneration(), 500); + } else if (nextProgress.state === "error") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError(nextProgress.error || nextProgress.message || "Generation failed."); + } else if (nextProgress.state === "cancelled") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + } + }, 250); + } catch (error) { + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError(error instanceof Error ? error.message : "Generation failed."); + } + }; + + const renderParam = (param: AIWorkflowParam) => { + const value = params[param.key]; + const isRangeParam = RANGE_PARAM_KEYS.has(param.key); + + if (param.type === "textarea") { + return ( +