From bf8cb9f3472c298378034ae3a83d71a359ba4c74 Mon Sep 17 00:00:00 2001 From: Sourav Das Date: Sun, 24 May 2026 08:24:17 +0530 Subject: [PATCH 1/2] chore: Implemented stable audio, testing with ace step --- CMakeLists.txt | 22 + Source/AITrackEngine.cpp | 192 ++- Source/AITrackEngine.h | 16 +- Source/AudioEngine.cpp | 122 +- Source/AudioEngine.h | 1 + Source/MainComponent.cpp | 50 +- Source/StemSeparator.cpp | 729 ++++++++- Source/StemSeparator.h | 23 +- docs/stable_audio_3_integration_plan.md | 148 ++ frontend/src/App.tsx | 14 +- .../__tests__/aiClipGenerationModal.test.ts | 10 + .../src/__tests__/aiGenerationStore.test.ts | 130 ++ .../__tests__/aiToolsFeatureInstaller.test.ts | 58 +- .../src/__tests__/aiWorkflowParams.test.ts | 119 ++ .../__tests__/interactionSafetyGuards.test.ts | 7 + .../__tests__/timelineDragAxisLock.test.ts | 55 + .../src/components/AIClipGenerationModal.tsx | 813 ++++++++++ frontend/src/components/AITrackHeader.tsx | 80 +- frontend/src/components/AIWorkflowModal.tsx | 605 ++++--- frontend/src/components/AiToolsSetupModal.tsx | 1391 ++++++++--------- frontend/src/components/ContextMenu.tsx | 31 +- frontend/src/components/Timeline.tsx | 189 ++- frontend/src/data/aiWorkflows.ts | 600 ++++++- frontend/src/services/NativeBridge.ts | 93 +- frontend/src/store/actions/project.ts | 36 +- frontend/src/store/actions/transport.ts | 92 +- frontend/src/store/actions/uiState.ts | 60 + frontend/src/store/useDAWStore.ts | 371 ++++- .../src/utils/globalShortcutDispatcher.ts | 11 + frontend/src/utils/timelineDragAxisLock.ts | 46 + frontend/src/utils/trackCreation.ts | 5 +- tools/ai-generation-ui-harness.mjs | 453 ++++++ tools/ai-runtime-install-plan-linux-cuda.json | 4 +- tools/ai-runtime-install-plan-linux-rocm.json | 4 +- .../ai-runtime-install-plan-windows-cuda.json | 4 +- ...runtime-install-plan-windows-directml.json | 4 +- tools/ai-runtime-requirements-linux-cuda.txt | 4 +- tools/ai-runtime-requirements-linux-rocm.txt | 4 +- tools/ai-runtime-requirements-linux.txt | 4 +- tools/ai-runtime-requirements-macos.txt | 4 +- .../ai-runtime-requirements-windows-cuda.txt | 4 +- ...-runtime-requirements-windows-directml.txt | 4 +- tools/ai_runtime_probe.py | 8 +- tools/generate_music.py | 263 +++- tools/install_ai_tools.py | 4 +- tools/openstudio_ace_runner.py | 3 +- tools/prepare_openstudio_ace_runtime.py | 198 +++ tools/stable_audio3_generate.py | 841 ++++++++++ tools/test_stable_audio3_generate.py | 100 ++ 49 files changed, 6738 insertions(+), 1291 deletions(-) create mode 100644 docs/stable_audio_3_integration_plan.md create mode 100644 frontend/src/__tests__/aiClipGenerationModal.test.ts create mode 100644 frontend/src/__tests__/aiGenerationStore.test.ts create mode 100644 frontend/src/__tests__/timelineDragAxisLock.test.ts create mode 100644 frontend/src/components/AIClipGenerationModal.tsx create mode 100644 frontend/src/utils/timelineDragAxisLock.ts create mode 100644 tools/ai-generation-ui-harness.mjs create mode 100644 tools/prepare_openstudio_ace_runtime.py create mode 100644 tools/stable_audio3_generate.py create mode 100644 tools/test_stable_audio3_generate.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a1d8d7c..eb8adf6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,6 +634,15 @@ if(EXISTS "${AI_MUSIC_GENERATION_SCRIPT}") ) endif() +set(AI_STABLE_AUDIO_GENERATION_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/stable_audio3_generate.py") +if(EXISTS "${AI_STABLE_AUDIO_GENERATION_SCRIPT}") + openstudio_copy_runtime_file( + "${AI_STABLE_AUDIO_GENERATION_SCRIPT}" + "scripts/stable_audio3_generate.py" + "Copying Stable Audio 3 generation helper script to output directory" + ) +endif() + set(AI_OPENSTUDIO_ACE_RUNNER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_runner.py") if(EXISTS "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}") openstudio_copy_runtime_file( @@ -644,6 +653,19 @@ if(EXISTS "${AI_OPENSTUDIO_ACE_RUNNER_SCRIPT}") endif() set(AI_OPENSTUDIO_ACE_BACKEND_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tools/openstudio_ace_backend") +set(AI_PREPARE_OPENSTUDIO_ACE_RUNTIME_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tools/prepare_openstudio_ace_runtime.py") +find_package(Python3 COMPONENTS Interpreter QUIET) +if(Python3_Interpreter_FOUND AND EXISTS "${AI_PREPARE_OPENSTUDIO_ACE_RUNTIME_SCRIPT}") + add_custom_command(TARGET OpenStudio PRE_BUILD + COMMAND "${Python3_EXECUTABLE}" + "${AI_PREPARE_OPENSTUDIO_ACE_RUNTIME_SCRIPT}" + "--destination" + "${AI_OPENSTUDIO_ACE_BACKEND_DIR}/vendor_runtime" + COMMENT "Preparing OpenStudio ACE vendor runtime from local ComfyUI source" + ) +else() + message(WARNING "OpenStudio ACE vendor runtime preflight could not run because Python3 or tools/prepare_openstudio_ace_runtime.py was not found.") +endif() if(EXISTS "${AI_OPENSTUDIO_ACE_BACKEND_DIR}") openstudio_copy_runtime_directory( "${AI_OPENSTUDIO_ACE_BACKEND_DIR}" diff --git a/Source/AITrackEngine.cpp b/Source/AITrackEngine.cpp index 4299181..618db75 100644 --- a/Source/AITrackEngine.cpp +++ b/Source/AITrackEngine.cpp @@ -3,6 +3,7 @@ namespace { constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; +constexpr auto kStableAudioModelId = "stable-audio-3-medium"; constexpr auto kReaderSleepMs = 50; constexpr auto kWorkerStartupTimeoutMs = 45000; constexpr auto kWorkerRequestTimeoutMs = 10000; @@ -246,6 +247,11 @@ juce::File AITrackEngine::getUserRuntimeRoot() const return getUserDataRoot().getChildFile("stem-runtime"); } +juce::File AITrackEngine::getStableAudioRuntimeRoot() const +{ + return getUserDataRoot().getChildFile("stable-audio-runtime"); +} + juce::File AITrackEngine::getMusicGenerationCheckpointRoot() const { return juce::File::getSpecialLocation(juce::File::userHomeDirectory) @@ -254,6 +260,13 @@ juce::File AITrackEngine::getMusicGenerationCheckpointRoot() const .getChildFile("checkpoints"); } +juce::File AITrackEngine::getStableAudioModelRoot() const +{ + return getUserDataRoot() + .getChildFile("models") + .getChildFile(kStableAudioModelId); +} + juce::File AITrackEngine::findPython() const { auto runtimePython = findPythonInRuntimeRoot(getUserRuntimeRoot()); @@ -295,6 +308,15 @@ juce::File AITrackEngine::findPython() const return {}; } +juce::File AITrackEngine::findStableAudioPython() const +{ + auto runtimePython = findPythonInRuntimeRoot(getStableAudioRuntimeRoot()); + if (runtimePython.existsAsFile()) + return runtimePython; + + return {}; +} + juce::File AITrackEngine::findScript() const { const auto runtimeDir = getApplicationRuntimeDirectory(); @@ -315,6 +337,26 @@ juce::File AITrackEngine::findScript() const return {}; } +juce::File AITrackEngine::findStableAudioScript() const +{ + const auto runtimeDir = getApplicationRuntimeDirectory(); + const auto packagedScript = runtimeDir.getChildFile("scripts").getChildFile("stable_audio3_generate.py"); + if (packagedScript.existsAsFile()) + return packagedScript; + + const auto bundledDevScript = runtimeDir.getChildFile("tools").getChildFile("stable_audio3_generate.py"); + if (bundledDevScript.existsAsFile()) + return bundledDevScript; + + const auto workingCopyScript = juce::File::getCurrentWorkingDirectory() + .getChildFile("tools") + .getChildFile("stable_audio3_generate.py"); + if (workingCopyScript.existsAsFile()) + return workingCopyScript; + + return {}; +} + void AITrackEngine::cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const { #if JUCE_WINDOWS @@ -330,10 +372,14 @@ void AITrackEngine::cleanupLegacyWorkerProcesses(const juce::File& python, const "Get-CimInstance Win32_Process | " "Where-Object { $_.CommandLine -like '*--worker*' " " -and $_.CommandLine -like '*" + escapedScriptPath + "*' " - " -and $_.CommandLine -like '*" + escapedPythonPath + "*' } | " + " -and (" + " $_.CommandLine -like '*" + escapedPythonPath + "*' " + " -or $_.CommandLine -like '*stable_audio3_generate.py*' " + " -or $_.CommandLine -like '*generate_music.py*'" + " ) } | " "ForEach-Object { " " & taskkill /PID $_.ProcessId /F /T 2>$null | Out-Null; " - " Write-Output ('stopped legacy ACE-Step worker pid ' + $_.ProcessId) " + " Write-Output ('stopped legacy AI generation worker pid ' + $_.ProcessId) " "}"); juce::ChildProcess cleanup; @@ -415,9 +461,12 @@ bool AITrackEngine::waitForWorkerReady(int timeoutMs) if (currentProgress_.error.isEmpty()) { + const auto modelLabel = currentProgress_.modelId == kStableAudioModelId + ? juce::String("Stable Audio 3") + : juce::String("ACE-Step"); const auto message = workerExitedBeforeReady - ? appendProcessDetailsLocked("ACE-Step worker exited before reporting ready.") - : appendProcessDetailsLocked("ACE-Step worker did not become ready in time."); + ? appendProcessDetailsLocked(modelLabel + " worker exited before reporting ready.") + : appendProcessDetailsLocked(modelLabel + " worker did not become ready in time."); setProgressErrorLocked(workerExitedBeforeReady ? "worker_start_failed" : "worker_start_timeout", message, @@ -471,7 +520,7 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo if (workerPidToKill > 0) { juce::String killOutput; - juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process tree pid=" + juce::Logger::writeToLog("AITrackEngine: stopping AI generation child process tree pid=" + juce::String(workerPidToKill)); killedTree = killWindowsProcessTree(workerPidToKill, &killOutput); if (killOutput.isNotEmpty()) @@ -481,7 +530,7 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo if (! killedTree && processToKill->isRunning()) { - juce::Logger::writeToLog("AITrackEngine: stopping ACE-Step child process"); + juce::Logger::writeToLog("AITrackEngine: stopping AI generation child process"); processToKill->kill(); } } @@ -501,8 +550,10 @@ void AITrackEngine::stopWorkerSession(bool clearProgress, bool userCancelled, bo } } -bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce::File& script) +bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce::File& script, const juce::String& modelId) { + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); const auto expectedScriptVersion = computeScriptVersion(script); { const juce::ScopedLock sl(lock_); @@ -524,13 +575,21 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: command.add(python.getFullPathName()); command.add(script.getFullPathName()); command.add("--worker"); - command.add("--checkpoint-root"); - command.add(getMusicGenerationCheckpointRoot().getFullPathName()); - command.add("--music-gen-model"); - command.add(kPinnedMusicGenerationModelId); + if (isStableAudio) + { + command.add("--model-root"); + command.add(getStableAudioModelRoot().getFullPathName()); + } + else + { + command.add("--checkpoint-root"); + command.add(getMusicGenerationCheckpointRoot().getFullPathName()); + command.add("--music-gen-model"); + command.add(kPinnedMusicGenerationModelId); + } const auto commandLine = buildCommandLineForLog(command); - juce::Logger::writeToLog("AITrackEngine: launching persistent ACE-Step worker: " + commandLine + juce::Logger::writeToLog("AITrackEngine: launching persistent " + modelLabel + " worker: " + commandLine + " protocolVersion=" + juce::String(kWorkerProtocolVersion) + " expectedScriptVersion=" + expectedScriptVersion + " launchAttempt=" + juce::String(launchAttempt + 1)); @@ -550,7 +609,8 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: currentProgress_.state = "loading"; currentProgress_.progress = 0.02f; currentProgress_.phase = "starting_worker"; - currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.message = "Starting the " + modelLabel + " runtime session..."; + currentProgress_.modelId = modelId; currentProgress_.backend = "unknown"; currentProgress_.error.clear(); currentProgress_.runMode = "cold"; @@ -563,10 +623,13 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: currentProgress_.scriptVersion = expectedScriptVersion; currentProgress_.lastStdoutLine.clear(); currentProgress_.lastStderrLine.clear(); - currentProgress_.tracePath.clear(); - currentProgress_.failureDetail.clear(); + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.attemptMode.clear(); currentProgress_.lmBackend.clear(); currentProgress_.lmStage.clear(); + currentProgress_.tracePath.clear(); + currentProgress_.failureDetail.clear(); } if (! workerProcess_->start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) @@ -574,7 +637,7 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: const juce::ScopedLock sl(lock_); workerProcess_.reset(); setProgressErrorLocked("worker_start_failed", - "Failed to start the ACE-Step runtime session.", + "Failed to start the " + modelLabel + " runtime session.", "worker_start"); return false; } @@ -599,10 +662,13 @@ bool AITrackEngine::ensureWorkerAvailable(const juce::File& python, const juce:: return false; } -bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, +bool AITrackEngine::sendGenerateRequest(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile) { + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); int port = 0; juce::String requestId; { @@ -617,7 +683,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_connect_failed", - appendProcessDetailsLocked("ACE-Step worker did not provide a listening port."), + appendProcessDetailsLocked(modelLabel + " worker did not provide a listening port."), "worker_request"); return false; } @@ -627,13 +693,14 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_connect_failed", - appendProcessDetailsLocked("OpenStudio could not contact the ACE-Step runtime session."), + appendProcessDetailsLocked("OpenStudio could not contact the " + modelLabel + " runtime session."), "worker_request"); return false; } auto request = std::make_unique(); request->setProperty("command", "generate"); + request->setProperty("modelId", modelId); request->setProperty("workflow", workflowId); request->setProperty("params", paramsJson); request->setProperty("output", outputFile.getFullPathName()); @@ -649,7 +716,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, return false; setProgressErrorLocked("worker_request_failed", - appendProcessDetailsLocked("OpenStudio could not fully submit the generation request to the ACE-Step session."), + appendProcessDetailsLocked("OpenStudio could not fully submit the generation request to the " + modelLabel + " session."), "worker_protocol"); return false; } @@ -668,7 +735,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, return false; setProgressErrorLocked("worker_request_timeout", - appendProcessDetailsLocked("ACE-Step did not acknowledge the generation request in time."), + appendProcessDetailsLocked(modelLabel + " did not acknowledge the generation request in time."), "worker_protocol"); return false; } @@ -678,10 +745,10 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { auto error = object != nullptr ? object->getProperty("error").toString() - : "ACE-Step returned an invalid worker response."; + : modelLabel + " returned an invalid worker response."; if (error.isEmpty()) - error = "ACE-Step rejected the generation request."; + error = modelLabel + " rejected the generation request."; const juce::ScopedLock sl(lock_); if (cancelRequested_) @@ -704,7 +771,7 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); setProgressErrorLocked("worker_protocol_failed", - appendProcessDetailsLocked("ACE-Step acknowledged the request with a mismatched protocol or request id."), + appendProcessDetailsLocked(modelLabel + " acknowledged the request with a mismatched protocol or request id."), "worker_protocol"); return false; } @@ -718,11 +785,15 @@ bool AITrackEngine::sendGenerateRequest(const juce::String& workflowId, void AITrackEngine::launchGenerationTask(const juce::File& python, const juce::File& script, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile) { - if (! ensureWorkerAvailable(python, script)) + const auto isStableAudio = modelId == kStableAudioModelId; + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); + + if (! ensureWorkerAvailable(python, script, modelId)) { const juce::ScopedLock sl(lock_); generationActive_ = false; @@ -749,7 +820,9 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, currentProgress_.state = "loading"; currentProgress_.progress = 0.03f; currentProgress_.phase = "submitting_request"; - currentProgress_.message = "Submitting the generation request to the ACE-Step session..."; + currentProgress_.message = "Submitting the generation request to the " + modelLabel + " session..."; + currentProgress_.modelId = modelId; + currentProgress_.workflowId = workflowId; currentProgress_.outputFile.clear(); currentProgress_.error.clear(); currentProgress_.elapsedMs = 0.0; @@ -759,9 +832,14 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, currentProgress_.sessionMode = "persistent"; currentProgress_.workerExitCode = 0; currentProgress_.failureKind.clear(); + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.attemptMode.clear(); + currentProgress_.lmBackend.clear(); + currentProgress_.lmStage.clear(); } - if (! sendGenerateRequest(workflowId, paramsJson, outputFile)) + if (! sendGenerateRequest(modelId, workflowId, paramsJson, outputFile)) { const juce::ScopedLock sl(lock_); generationActive_ = false; @@ -774,7 +852,8 @@ void AITrackEngine::launchGenerationTask(const juce::File& python, } } -bool AITrackEngine::startGeneration(const juce::String& workflowId, +bool AITrackEngine::startGeneration(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputDir) { @@ -786,14 +865,16 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, return false; } - const auto python = findPython(); - const auto script = findScript(); + const auto isStableAudio = modelId == kStableAudioModelId; + const auto python = isStableAudio ? findStableAudioPython() : findPython(); + const auto script = isStableAudio ? findStableAudioScript() : findScript(); + const auto modelLabel = isStableAudio ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); if (! python.existsAsFile() || ! script.existsAsFile()) { const juce::ScopedLock sl(lock_); setProgressErrorLocked("runtime_missing", - "AI runtime is not ready. Install AI Tools first.", + modelLabel + " runtime is not ready. Install AI Tools first.", "runtime_missing"); return false; } @@ -811,7 +892,8 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, { const juce::ScopedLock sl(lock_); currentOutputFile_ = outputDir.getChildFile( - "generated_music_" + createSafeMusicGenerationTimestamp() + ".wav"); + (isStableAudio ? "generated_stable_audio_" : "generated_music_") + + createSafeMusicGenerationTimestamp() + ".wav"); outputFile = currentOutputFile_; generationActive_ = true; cancelRequested_ = false; @@ -823,7 +905,10 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, currentProgress_.state = "loading"; currentProgress_.progress = 0.01f; currentProgress_.phase = "starting"; - currentProgress_.message = "Starting the ACE-Step runtime session..."; + currentProgress_.message = "Starting the " + modelLabel + " runtime session..."; + currentProgress_.modelId = modelId; + currentProgress_.workflowId = workflowId; + currentProgress_.sourceClipId.clear(); currentProgress_.outputFile.clear(); currentProgress_.error.clear(); currentProgress_.elapsedMs = 0.0; @@ -836,18 +921,22 @@ bool AITrackEngine::startGeneration(const juce::String& workflowId, currentProgress_.lastStdoutLine.clear(); currentProgress_.lastStderrLine.clear(); currentProgress_.statusNote.clear(); - currentProgress_.attemptMode = "lm_dit"; + currentProgress_.attemptMode = isStableAudio ? juce::String() : juce::String("lm_dit"); currentProgress_.attemptIndex = 1; currentProgress_.protocolVersion = kWorkerProtocolVersion; currentProgress_.scriptVersion = expectedScriptVersion_; currentProgress_.requestId = currentRequestId_; currentProgress_.priorFailure.clear(); currentProgress_.lastProgressAgeMs = 0.0; + currentProgress_.runtimeProfile.clear(); + currentProgress_.lmModel.clear(); + currentProgress_.lmBackend.clear(); + currentProgress_.lmStage.clear(); } - generationThread_ = std::thread([this, python, script, workflowId, paramsJson, outputFile]() + generationThread_ = std::thread([this, python, script, modelId, workflowId, paramsJson, outputFile]() { - launchGenerationTask(python, script, workflowId, paramsJson, outputFile); + launchGenerationTask(python, script, modelId, workflowId, paramsJson, outputFile); }); return true; @@ -906,11 +995,14 @@ void AITrackEngine::parseOutputLine(const juce::String& line) if (workerProtocolVersion_ != kWorkerProtocolVersion || (! expectedScriptVersion_.isEmpty() && workerScriptVersion_ != expectedScriptVersion_)) { + const auto modelLabel = currentProgress_.modelId == kStableAudioModelId + ? juce::String("Stable Audio 3") + : juce::String("ACE-Step"); workerProtocolRejected_ = true; workerReady_ = false; workerPort_ = 0; setProgressErrorLocked("worker_protocol_failed", - appendProcessDetailsLocked("ACE-Step worker protocol or script version mismatch."), + appendProcessDetailsLocked(modelLabel + " worker protocol or script version mismatch."), "worker_protocol"); currentProgress_.statusNote = "Rejecting stale worker session before generation starts."; juce::Logger::writeToLog("AITrackEngine: rejecting worker ready handshake due to version mismatch" @@ -923,11 +1015,13 @@ void AITrackEngine::parseOutputLine(const juce::String& line) workerReady_ = true; workerPort_ = static_cast (double (obj->getProperty("port"))); + const auto backend = obj->getProperty("backend").toString(); + const auto readyLabel = backend == "stable-audio-3" ? juce::String("Stable Audio 3") : juce::String("ACE-Step"); currentProgress_.state = "idle"; currentProgress_.progress = 0.0f; currentProgress_.phase = "worker_ready"; - currentProgress_.message = "ACE-Step runtime session is ready."; - currentProgress_.backend = obj->getProperty("backend").toString(); + currentProgress_.message = readyLabel + " runtime session is ready."; + currentProgress_.backend = backend; currentProgress_.sessionMode = obj->hasProperty("sessionMode") ? obj->getProperty("sessionMode").toString() : "persistent"; @@ -987,6 +1081,12 @@ void AITrackEngine::parseOutputLine(const juce::String& line) currentProgress_.backend = obj->getProperty("backend").toString(); if (obj->hasProperty("outputFile")) currentProgress_.outputFile = obj->getProperty("outputFile").toString(); + if (obj->hasProperty("modelId")) + currentProgress_.modelId = obj->getProperty("modelId").toString(); + if (obj->hasProperty("workflowId")) + currentProgress_.workflowId = obj->getProperty("workflowId").toString(); + if (obj->hasProperty("sourceClipId")) + currentProgress_.sourceClipId = obj->getProperty("sourceClipId").toString(); if (obj->hasProperty("error")) currentProgress_.error = obj->getProperty("error").toString(); if (obj->hasProperty("elapsedMs")) @@ -1265,6 +1365,7 @@ void AITrackEngine::stopWorker(bool clearProgress, bool userCancelled) AIGenerationProgress AITrackEngine::pollProgress() { bool shouldStopForDecodeStall = false; + bool shouldReleaseTerminalWorker = false; { const juce::ScopedLock sl(lock_); @@ -1324,10 +1425,23 @@ AIGenerationProgress AITrackEngine::pollProgress() currentProgress_.workerExitCode = workerExitCode_; currentProgress_.lastStdoutLine = lastStdoutLine_; currentProgress_.lastStderrLine = lastStderrLine_; + + const auto inTerminalState = currentProgress_.state == "done" + || currentProgress_.state == "error" + || currentProgress_.state == "cancelled"; + if (inTerminalState + && ! generationActive_ + && workerProcess_ != nullptr + && workerProcess_->isRunning()) + { + shouldReleaseTerminalWorker = true; + } } if (shouldStopForDecodeStall) stopWorkerSession(false, false, false); + else if (shouldReleaseTerminalWorker) + stopWorkerSession(false, false, false); const juce::ScopedLock sl(lock_); return currentProgress_; diff --git a/Source/AITrackEngine.h b/Source/AITrackEngine.h index 05fcacb..6d935a4 100644 --- a/Source/AITrackEngine.h +++ b/Source/AITrackEngine.h @@ -14,6 +14,9 @@ struct AIGenerationProgress juce::String message; juce::String backend { "unknown" }; juce::String outputFile; + juce::String modelId; + juce::String workflowId; + juce::String sourceClipId; juce::String error; double elapsedMs = 0.0; double heartbeatTs = 0.0; @@ -47,7 +50,8 @@ class AITrackEngine AITrackEngine() = default; ~AITrackEngine(); - bool startGeneration(const juce::String& workflowId, + bool startGeneration(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputDir); @@ -58,16 +62,22 @@ class AITrackEngine private: juce::File getUserDataRoot() const; juce::File getUserRuntimeRoot() const; + juce::File getStableAudioRuntimeRoot() const; juce::File getMusicGenerationCheckpointRoot() const; + juce::File getStableAudioModelRoot() const; juce::File findPython() const; + juce::File findStableAudioPython() const; juce::File findScript() const; + juce::File findStableAudioScript() const; void cleanupLegacyWorkerProcesses(const juce::File& python, const juce::File& script) const; - bool ensureWorkerAvailable(const juce::File& python, const juce::File& script); - bool sendGenerateRequest(const juce::String& workflowId, + bool ensureWorkerAvailable(const juce::File& python, const juce::File& script, const juce::String& modelId); + bool sendGenerateRequest(const juce::String& modelId, + const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile); void launchGenerationTask(const juce::File& python, const juce::File& script, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJson, const juce::File& outputFile); diff --git a/Source/AudioEngine.cpp b/Source/AudioEngine.cpp index 80da129..53144bd 100644 --- a/Source/AudioEngine.cpp +++ b/Source/AudioEngine.cpp @@ -3920,10 +3920,13 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha if (track->hasActiveARA()) anyActiveARAInCallback = true; - // Solo logic: if any track is soloed, skip non-soloed tracks entirely - // (recording still works because record-armed tracks should also be soloed, - // and in practice users don't solo-off a track they're actively recording) - if (anySoloed && !track->getSolo()) + const bool excludedBySolo = anySoloed && !track->getSolo(); + const bool hasActiveAudioRecording = isRecordMode.load() && audioRecorder.isRecording(trackId); + const bool needsRecordingPath = track->getRecordArmed() || hasActiveAudioRecording; + + // Solo should silence non-soloed tracks, but it must not prevent an + // armed/active track from reaching the recorder. + if (excludedBySolo && !needsRecordingPath) { track->resetRMS(); continue; @@ -3938,7 +3941,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha && !staticMuteBlocksInput && (track->getRecordArmed() || track->getInputMonitoring() - || audioRecorder.isRecording(trackId)); + || hasActiveAudioRecording); const double blockStartTimeSeconds = currentSamplePosition / currentSampleRate; const bool shouldProcessMidi = track->needsProcessing(blockStartTimeSeconds, numSamples, currentSampleRate, isPlaying.load()); @@ -4049,7 +4052,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha // ========== RECORD RAW AUDIO (BEFORE FX) ========== // Write to recorder if transport is playing AND in record mode // This captures the raw input BEFORE any FX processing - if (isPlaying && isRecordMode && audioRecorder.isRecording(trackId)) + if (isPlaying && isRecordMode.load() && hasActiveAudioRecording) { // Punch recording: only write audio within punch range bool shouldWrite = true; @@ -4144,7 +4147,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha } // ========== SEND MIXING: fill destination track send accum buffers ========== - if (!trackEntry.sends.empty()) + if (!excludedBySolo && !trackEntry.sends.empty()) { const auto& preFaderBuffer = track->getPreFaderBuffer(); for (const auto& send : trackEntry.sends) @@ -4194,7 +4197,7 @@ void AudioEngine::audioDeviceIOCallbackWithContext (const float* const* inputCha track->sendMIDIToOutput(midiMessages, currentSampleRate, track->getMute()); // Mix track output to device outputs (only if master send is enabled) - if (track->getMasterSendEnabled()) + if (!excludedBySolo && track->getMasterSendEnabled()) { int outStart = track->getOutputStartChannel(); int outCount = track->getOutputChannelCount(); @@ -17009,6 +17012,7 @@ void AudioEngine::cancelAiToolsInstall() } juce::var AudioEngine::startAIGeneration(const juce::String& trackId, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJSON) { @@ -17026,51 +17030,71 @@ juce::var AudioEngine::startAIGeneration(const juce::String& trackId, auto aiToolsStatus = stemSeparator.getAiToolsStatus(); if (auto* statusObject = aiToolsStatus.getDynamicObject()) { - const auto musicGenerationReady = static_cast(statusObject->getProperty("musicGenerationReady")); - const auto layoutValid = static_cast(statusObject->getProperty("musicGenerationLayoutValid")); - const auto performanceReady = ! statusObject->hasProperty("musicGenerationPerformanceReady") - || static_cast(statusObject->getProperty("musicGenerationPerformanceReady")); - const auto availableProfilesVar = statusObject->getProperty("musicGenerationAvailableProfiles"); - auto hasNativeMusicProfile = true; - if (auto* availableProfilesArray = availableProfilesVar.getArray()) + if (modelId == "stable-audio-3-medium") + { + auto stableReady = false; + juce::String stableMessage = "Stable Audio 3 Medium is not set up yet."; + if (auto* musicModels = statusObject->getProperty("musicModels").getDynamicObject()) + { + if (auto* stableModel = musicModels->getProperty("stable-audio-3-medium").getDynamicObject()) + { + stableReady = static_cast(stableModel->getProperty("ready")); + stableMessage = stableModel->getProperty("message").toString(); + } + } + + if (! stableReady) + { + result->setProperty("started", false); + result->setProperty("error", stableMessage.isNotEmpty() + ? stableMessage + : "Import Stable Audio 3 Medium from AI Tools Setup before generating."); + return juce::var(result.release()); + } + } + else { - if (! availableProfilesArray->isEmpty()) + const auto musicGenerationReady = static_cast(statusObject->getProperty("musicGenerationReady")); + const auto layoutValid = static_cast(statusObject->getProperty("musicGenerationLayoutValid")); + const auto availableProfilesVar = statusObject->getProperty("musicGenerationAvailableProfiles"); + auto hasNativeMusicProfile = true; + if (auto* availableProfilesArray = availableProfilesVar.getArray()) { - hasNativeMusicProfile = false; - for (const auto& profile : *availableProfilesArray) + if (! availableProfilesArray->isEmpty()) { - if (profile.toString() == "native-xl-turbo") + hasNativeMusicProfile = false; + for (const auto& profile : *availableProfilesArray) { - hasNativeMusicProfile = true; - break; + if (profile.toString() == "native-xl-turbo") + { + hasNativeMusicProfile = true; + break; + } } } } - } - if (! musicGenerationReady || ! layoutValid || ! performanceReady || ! hasNativeMusicProfile) - { - const auto musicGenerationStatusMessage = statusObject->getProperty("musicGenerationStatusMessage").toString(); - const auto musicGenerationPerformanceStatusMessage = statusObject->getProperty("musicGenerationPerformanceStatusMessage").toString(); - const auto statusMessage = statusObject->getProperty("message").toString(); - const auto errorMessage = statusObject->getProperty("error").toString(); - const auto modelId = statusObject->getProperty("musicGenerationModelId").toString(); - const auto checkpointRoot = statusObject->getProperty("musicGenerationCheckpointRoot").toString(); - result->setProperty("started", false); - result->setProperty( - "error", - errorMessage.isNotEmpty() - ? errorMessage - : (! hasNativeMusicProfile) - ? "The Native XL Turbo ACE-Step profile is still missing required music-generation assets. Retry AI Tools install to finish setup." - : musicGenerationPerformanceStatusMessage.isNotEmpty() - ? musicGenerationPerformanceStatusMessage - : musicGenerationStatusMessage.isNotEmpty() - ? musicGenerationStatusMessage - : (! layoutValid && modelId.isNotEmpty() && checkpointRoot.isNotEmpty()) - ? "Pinned ACE-Step model " + modelId + " is not ready in " + checkpointRoot + "." - : statusMessage); - return juce::var(result.release()); + if (! musicGenerationReady || ! layoutValid || ! hasNativeMusicProfile) + { + const auto musicGenerationStatusMessage = statusObject->getProperty("musicGenerationStatusMessage").toString(); + const auto statusMessage = statusObject->getProperty("message").toString(); + const auto errorMessage = statusObject->getProperty("error").toString(); + const auto aceModelId = statusObject->getProperty("musicGenerationModelId").toString(); + const auto checkpointRoot = statusObject->getProperty("musicGenerationCheckpointRoot").toString(); + result->setProperty("started", false); + result->setProperty( + "error", + errorMessage.isNotEmpty() + ? errorMessage + : (! hasNativeMusicProfile) + ? "The Native XL Turbo ACE-Step profile is still missing required music-generation assets. Retry AI Tools install to finish setup." + : musicGenerationStatusMessage.isNotEmpty() + ? musicGenerationStatusMessage + : (! layoutValid && aceModelId.isNotEmpty() && checkpointRoot.isNotEmpty()) + ? "Pinned ACE-Step model " + aceModelId + " is not ready in " + checkpointRoot + "." + : statusMessage); + return juce::var(result.release()); + } } } @@ -17078,7 +17102,7 @@ juce::var AudioEngine::startAIGeneration(const juce::String& trackId, .getChildFile("OpenStudio") .getChildFile("generated-music"); - if (! aiTrackEngine.startGeneration(workflowId, paramsJSON, outputDir)) + if (! aiTrackEngine.startGeneration(modelId, workflowId, paramsJSON, outputDir)) { auto progress = aiTrackEngine.pollProgress(); result->setProperty("started", false); @@ -17102,6 +17126,12 @@ juce::var AudioEngine::getAIGenerationProgress() obj->setProperty("phase", progress.phase); obj->setProperty("message", progress.message); obj->setProperty("backend", progress.backend); + if (progress.modelId.isNotEmpty()) + obj->setProperty("modelId", progress.modelId); + if (progress.workflowId.isNotEmpty()) + obj->setProperty("workflowId", progress.workflowId); + if (progress.sourceClipId.isNotEmpty()) + obj->setProperty("sourceClipId", progress.sourceClipId); obj->setProperty("elapsedMs", progress.elapsedMs); obj->setProperty("heartbeatTs", progress.heartbeatTs); if (progress.phaseProgress >= 0.0) diff --git a/Source/AudioEngine.h b/Source/AudioEngine.h index e3e6e81..aed5138 100644 --- a/Source/AudioEngine.h +++ b/Source/AudioEngine.h @@ -524,6 +524,7 @@ class AudioEngine : public juce::AudioIODeviceCallback, void cancelStemSeparation(); void cancelAiToolsInstall(); juce::var startAIGeneration(const juce::String& trackId, + const juce::String& modelId, const juce::String& workflowId, const juce::String& paramsJSON); juce::var getAIGenerationProgress(); diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index ee745d6..1d328b7 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -2514,6 +2514,50 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::URL(args[0].toString()).launchInDefaultBrowser()); }) + .withNativeFunction ("browseForFile", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const auto title = args.size() > 0 && args[0].isString() + ? args[0].toString() + : juce::String("Select File"); + const auto filters = args.size() > 1 && args[1].isString() + ? args[1].toString() + : juce::String("*"); + auto initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + + fileChooser = std::make_unique( + title, + initialDir, + filters.isNotEmpty() ? filters : juce::String("*"), + true); + + const auto chooserFlags = juce::FileBrowserComponent::openMode + | juce::FileBrowserComponent::canSelectFiles; + fileChooser->launchAsync(chooserFlags, [completion] (const juce::FileChooser& fc) { + auto result = fc.getResult(); + completion(result.existsAsFile() ? result.getFullPathName() : juce::String()); + }); + }) + .withNativeFunction ("browseForFolder", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { + const auto title = args.size() > 0 && args[0].isString() + ? args[0].toString() + : juce::String("Select Folder"); + auto initialDir = juce::File::getSpecialLocation(juce::File::userHomeDirectory) + .getChildFile("Downloads"); + if (! initialDir.isDirectory()) + initialDir = juce::File::getSpecialLocation(juce::File::userDocumentsDirectory); + + fileChooser = std::make_unique( + title, + initialDir, + "*", + true); + + const auto chooserFlags = juce::FileBrowserComponent::openMode + | juce::FileBrowserComponent::canSelectDirectories; + fileChooser->launchAsync(chooserFlags, [completion] (const juce::FileChooser& fc) { + auto result = fc.getResult(); + completion(result.isDirectory() ? result.getFullPathName() : juce::String()); + }); + }) // ========== Project Save/Load (F2) ========== .withNativeFunction ("showSaveDialog", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { // Show native save file dialog @@ -6155,8 +6199,10 @@ MainComponent::MainComponent(AudioEngine& audioEngineIn, completion(juce::var()); }) .withNativeFunction ("startAIGeneration", [this] (const juce::Array& args, juce::WebBrowserComponent::NativeFunctionCompletion completion) { - if (args.size() >= 3) - completion(audioEngine.startAIGeneration(args[0].toString(), args[1].toString(), args[2].toString())); + if (args.size() >= 4) + completion(audioEngine.startAIGeneration(args[0].toString(), args[1].toString(), args[2].toString(), args[3].toString())); + else if (args.size() >= 3) + completion(audioEngine.startAIGeneration(args[0].toString(), "ace-step-v15-xl-turbo", args[1].toString(), args[2].toString())); else completion(juce::var()); }) diff --git a/Source/StemSeparator.cpp b/Source/StemSeparator.cpp index 94ba435..5334594 100644 --- a/Source/StemSeparator.cpp +++ b/Source/StemSeparator.cpp @@ -10,6 +10,8 @@ constexpr auto kStemModelName = "BS-Roformer-SW.ckpt"; constexpr auto kPinnedMusicGenerationModelId = "acestep-v15-xl-turbo"; constexpr auto kPinnedMusicGenerationModelRepoId = "ACE-Step/acestep-v15-xl-turbo"; constexpr auto kPinnedMusicGenerationSharedRepoId = "ACE-Step/Ace-Step1.5"; +constexpr auto kStableAudioModelId = "stable-audio-3-medium"; +constexpr auto kStableAudioAttribution = "Powered by Stability AI"; constexpr auto kFeatureStemSeparation = "stemSeparation"; constexpr auto kFeatureAudioGeneration = "audioGeneration"; constexpr juce::int64 kMinStemSystemRamMb = 8 * 1024; @@ -67,6 +69,23 @@ juce::StringArray varToStringArray (const juce::var& value) return result; } +juce::StringArray getStableAudioRequiredRelativePaths() +{ + return { + "model.safetensors", + "model_config.json", + "LICENSE.md", + "LICENSE_GEMMA.md", + "NOTICE", + "t5gemma-b-b-ul2/model.safetensors", + "t5gemma-b-b-ul2/config.json", + "t5gemma-b-b-ul2/tokenizer.json", + "t5gemma-b-b-ul2/tokenizer.model", + "t5gemma-b-b-ul2/tokenizer_config.json", + "t5gemma-b-b-ul2/special_tokens_map.json", + }; +} + juce::String normaliseFeatureId (juce::String value) { value = value.trim(); @@ -262,6 +281,11 @@ juce::File StemSeparator::getUserRuntimeRoot() const return getUserDataRoot().getChildFile("stem-runtime"); } +juce::File StemSeparator::getStableAudioRuntimeRoot() const +{ + return getUserDataRoot().getChildFile("stable-audio-runtime"); +} + juce::File StemSeparator::getUserModelsDir() const { return getUserDataRoot().getChildFile("models"); @@ -275,6 +299,16 @@ juce::File StemSeparator::getMusicGenerationCheckpointRoot() const .getChildFile("checkpoints"); } +juce::File StemSeparator::getStableAudioModelRoot() const +{ + return getUserModelsDir().getChildFile(kStableAudioModelId); +} + +juce::File StemSeparator::findStableAudioPython() const +{ + return findPythonInRuntimeRoot(getStableAudioRuntimeRoot()); +} + juce::File StemSeparator::getAiToolsInstallLogFile() const { return getUserDataRoot().getChildFile("logs").getChildFile("AiToolsInstall.log"); @@ -694,6 +728,28 @@ bool StemSeparator::hasRequiredModel(const juce::File& modelsDir) const return modelsDir.isDirectory() && modelsDir.getChildFile(kStemModelName).existsAsFile(); } +juce::StringArray StemSeparator::getMissingStableAudioFiles (const juce::File& modelRoot) const +{ + juce::StringArray missing; + if (! modelRoot.isDirectory()) + { + missing.add(modelRoot.getFullPathName()); + return missing; + } + + for (const auto& relativePath : getStableAudioRequiredRelativePaths()) + { + if (! modelRoot.getChildFile(relativePath).existsAsFile()) + missing.add(relativePath); + } + return missing; +} + +bool StemSeparator::isStableAudioModelFolderValid (const juce::File& modelRoot) const +{ + return getMissingStableAudioFiles(modelRoot).isEmpty(); +} + bool StemSeparator::isExternalPythonFallbackEnabled() const { #if OPENSTUDIO_ENABLE_EXTERNAL_PYTHON_AI_FALLBACK @@ -703,6 +759,15 @@ bool StemSeparator::isExternalPythonFallbackEnabled() const #endif } +bool StemSeparator::hasStableAudioRuntime() const +{ + const auto python = findStableAudioPython(); + return python.existsAsFile() + && getStableAudioRuntimeRoot() + .getChildFile(".openstudio-stable-audio-ready") + .existsAsFile(); +} + StemSeparator::InstallOptions StemSeparator::parseInstallOptions (const juce::String& optionsJson, bool legacyConfirmed) const { InstallOptions options; @@ -723,9 +788,18 @@ StemSeparator::InstallOptions StemSeparator::parseInstallOptions (const juce::St if (parsed.hasProperty("requestedFeature")) options.requestedFeature = normaliseFeatureId(parsed["requestedFeature"].toString()); + if (parsed.hasProperty("modelId")) + options.modelId = parsed["modelId"].toString().trim(); + if (parsed.hasProperty("selectedFeatures")) options.selectedFeatures = normaliseFeatureArray(parsed["selectedFeatures"]); + if (parsed.hasProperty("stableAudioModelPath")) + options.stableAudioModelPath = parsed["stableAudioModelPath"].toString(); + + if (parsed.hasProperty("stableAudioLicenseAccepted")) + options.stableAudioLicenseAccepted = static_cast(parsed["stableAudioLicenseAccepted"]); + if (options.selectedFeatures.isEmpty() && options.requestedFeature.isNotEmpty()) options.selectedFeatures.add(options.requestedFeature); @@ -897,7 +971,6 @@ juce::var StemSeparator::buildFeatureStatusVar (const AiToolsStatus& status, con const auto stemReady = status.scriptAvailable && status.runtimeInstalled && status.modelInstalled; const auto audioReady = status.musicGenerationReady && status.musicGenerationLayoutValid - && status.musicGenerationPerformanceReady && hasNativeMusicProfile(status); auto features = std::make_unique(); @@ -961,7 +1034,6 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File status.progress = 1.0f; const auto musicGenerationFullyReady = status.musicGenerationReady && status.musicGenerationLayoutValid - && status.musicGenerationPerformanceReady && hasNativeMusicProfile(status); status.stepLabel = musicGenerationFullyReady ? "AI tools are ready." : "Stem separation is ready."; status.stepIndex = 0; @@ -989,6 +1061,14 @@ StemSeparator::AiToolsStatus StemSeparator::buildAiToolsStatus (const juce::File status.activityLines.add(status.stepLabel); if (status.message != status.stepLabel) status.activityLines.add(status.message); + if (musicGenerationFullyReady + && ! status.musicGenerationPerformanceReady + && status.musicGenerationPerformanceStatusMessage.isNotEmpty()) + { + status.statusWarning = status.musicGenerationPerformanceStatusMessage; + status.statusWarningCode = "music_generation_acceleration_warning"; + status.activityLines.add(status.statusWarning); + } if (status.installedFeatures.isEmpty()) status.installedFeatures.add(kFeatureStemSeparation); if (musicGenerationFullyReady) @@ -1280,7 +1360,6 @@ void StemSeparator::scheduleStatusRefresh() refreshedStatus.installedFeatures.add(kFeatureStemSeparation); if (refreshedStatus.musicGenerationReady && refreshedStatus.musicGenerationLayoutValid - && refreshedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(refreshedStatus)) refreshedStatus.installedFeatures.add(kFeatureAudioGeneration); const auto refreshedHardware = probeHardwareStatus(); @@ -1288,7 +1367,6 @@ void StemSeparator::scheduleStatusRefresh() refreshedStatus.features = buildFeatureStatusVar(refreshedStatus, refreshedHardware); const auto musicGenerationFullyReady = refreshedStatus.musicGenerationReady && refreshedStatus.musicGenerationLayoutValid - && refreshedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(refreshedStatus); const auto isReconciliationRefresh = previousStatus.statusWarningCode == "reconciling_install_state"; @@ -1626,7 +1704,7 @@ bool StemSeparator::extractRuntimeArchive (const juce::File& archiveFile, return true; } -juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) +juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) const { auto obj = std::make_unique(); juce::Array supportedBackends; @@ -1709,6 +1787,55 @@ juce::var StemSeparator::aiToolsStatusToVar(const AiToolsStatus& status) obj->setProperty("requestedFeature", status.requestedFeature); obj->setProperty("hardware", status.hardware); obj->setProperty("features", status.features); + + auto musicModels = std::make_unique(); + const auto aceReady = status.musicGenerationReady + && status.musicGenerationLayoutValid + && hasNativeMusicProfile(status); + auto aceModel = std::make_unique(); + aceModel->setProperty("id", kPinnedMusicGenerationModelId); + aceModel->setProperty("label", "ACE-Step 1.5 XL Turbo"); + aceModel->setProperty("installed", status.musicGenerationReady && status.musicGenerationLayoutValid); + aceModel->setProperty("ready", aceReady); + aceModel->setProperty("compatible", true); + aceModel->setProperty("blocked", ! aceReady); + aceModel->setProperty("blockReason", aceReady ? juce::String() : status.musicGenerationStatusMessage); + aceModel->setProperty("message", aceReady ? "ACE-Step is ready." : status.musicGenerationStatusMessage); + aceModel->setProperty("modelPath", status.musicGenerationCheckpointRoot); + musicModels->setProperty(kPinnedMusicGenerationModelId, juce::var(aceModel.release())); + + const auto stableRoot = getStableAudioModelRoot(); + const auto stableMissing = getMissingStableAudioFiles(stableRoot); + const auto stableModelReady = stableMissing.isEmpty(); + const auto stableRuntimeReady = hasStableAudioRuntime(); + const auto stableReady = stableModelReady && stableRuntimeReady; + auto stableModel = std::make_unique(); + stableModel->setProperty("id", kStableAudioModelId); + stableModel->setProperty("label", "Stable Audio 3 Medium"); + stableModel->setProperty("installed", stableReady); + stableModel->setProperty("ready", stableReady); + stableModel->setProperty("compatible", true); + stableModel->setProperty("blocked", ! stableReady); + stableModel->setProperty("blockReason", + stableReady ? juce::String() + : stableModelReady ? "Stable Audio 3 runtime dependencies are not installed." + : "Stable Audio 3 Medium has not been imported."); + stableModel->setProperty("message", + stableReady ? "Stable Audio 3 Medium is ready." + : stableModelReady ? "Install the separate Stable Audio 3 runtime dependencies." + : "Import the Stable Audio 3 Medium Hugging Face snapshot."); + stableModel->setProperty("modelPath", stableRoot.getFullPathName()); + stableModel->setProperty("runtimePath", getStableAudioRuntimeRoot().getFullPathName()); + stableModel->setProperty("runtimeReady", stableRuntimeReady); + stableModel->setProperty("modelReady", stableModelReady); + stableModel->setProperty("attribution", kStableAudioAttribution); + stableModel->setProperty("licenseAccepted", stableModelReady); + juce::Array stableMissingArray; + for (const auto& missing : stableMissing) + stableMissingArray.add(missing); + stableModel->setProperty("missingFiles", stableMissingArray); + musicModels->setProperty(kStableAudioModelId, juce::var(stableModel.release())); + obj->setProperty("musicModels", juce::var(musicModels.release())); return juce::var(obj.release()); } @@ -1753,10 +1880,600 @@ juce::var StemSeparator::installAiTools (const juce::String& optionsJson) auto cachedStatus = getCachedAiToolsStatusSnapshot(); const bool musicGenerationFullyReady = cachedStatus.musicGenerationReady && cachedStatus.musicGenerationLayoutValid - && cachedStatus.musicGenerationPerformanceReady && hasNativeMusicProfile(cachedStatus); auto result = std::make_unique(); + + const bool stableAudioSetupRequested = installOptions.modelId == kStableAudioModelId + || installOptions.stableAudioModelPath.isNotEmpty(); + + if (stableAudioSetupRequested) + { + const auto stableSessionId = juce::Uuid().toString(); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_requested", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("modelId", kStableAudioModelId); + payload.setProperty("sourcePath", installOptions.stableAudioModelPath); + })); + + if (cachedStatus.installInProgress) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "install_already_running"); + })); + result->setProperty("started", false); + result->setProperty("error", "AI tools installation is already running."); + result->setProperty("status", aiToolsStatusToVar(cachedStatus)); + return juce::var(result.release()); + } + + if (installOptions.stableAudioModelPath.isEmpty()) + { + auto status = cachedStatus; + status.state = "error"; + status.progress = 0.0f; + status.installInProgress = false; + status.message = "Choose the downloaded Stable Audio 3 Medium snapshot folder before setup."; + status.error = status.message; + status.errorCode = "stable_audio_model_path_required"; + status.lastPhase = "stable_audio_import"; + status.terminalReason = status.errorCode; + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + { + const juce::ScopedLock lock(aiToolsStatusLock); + lastAiToolsStatus = status; + initialStatusPrepared = true; + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "missing_source_path"); + })); + result->setProperty("started", false); + result->setProperty("error", status.error); + result->setProperty("status", aiToolsStatusToVar(status)); + return juce::var(result.release()); + } + + if (! installOptions.stableAudioLicenseAccepted) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_rejected", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "license_not_accepted"); + })); + result->setProperty("started", false); + result->setProperty("error", "Stable Audio 3 setup requires accepting the Stability AI and Gemma license notices before import."); + result->setProperty("status", aiToolsStatusToVar(cachedStatus)); + return juce::var(result.release()); + } + + const juce::File sourceRoot(installOptions.stableAudioModelPath); + const auto missingFiles = getMissingStableAudioFiles(sourceRoot); + if (! missingFiles.isEmpty()) + { + auto status = cachedStatus; + status.state = "error"; + status.progress = 0.0f; + status.installInProgress = false; + status.message = "The selected Stable Audio 3 folder is missing required files."; + status.error = "Missing files: " + missingFiles.joinIntoString(", "); + status.errorCode = "stable_audio_model_layout_invalid"; + status.lastPhase = "stable_audio_import"; + status.terminalReason = status.errorCode; + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + { + const juce::ScopedLock lock(aiToolsStatusLock); + lastAiToolsStatus = status; + initialStatusPrepared = true; + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_source_layout_invalid", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", sourceRoot.getFullPathName()); + juce::Array missing; + for (const auto& file : missingFiles) + missing.add(file); + payload.setProperty("missingFiles", missing); + })); + result->setProperty("started", false); + result->setProperty("error", status.error); + result->setProperty("status", aiToolsStatusToVar(status)); + return juce::var(result.release()); + } + + aiToolsCancelRequested = false; + aiToolsInstallWorkInProgress = true; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = 0.05f; + status.installInProgress = true; + status.message = "Importing Stable Audio 3 Medium snapshot..."; + status.stepLabel = "Importing Stable Audio 3 Medium"; + status.lastPhase = "stable_audio_import"; + status.error.clear(); + status.errorCode.clear(); + status.activityLines.clear(); + status.activityLines.add("Valid Stable Audio 3 source: " + sourceRoot.getFullPathName()); + status.activityLines.add("Copying snapshot into the managed OpenStudio model folder."); + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + status.installSessionId = stableSessionId; + status.selectedFeatures.clear(); + status.selectedFeatures.add(kFeatureAudioGeneration); + status.requestedFeatures = status.selectedFeatures; + status.requestedFeature = kFeatureAudioGeneration; + }); + + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_import_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", sourceRoot.getFullPathName()); + payload.setProperty("destinationPath", getStableAudioModelRoot().getFullPathName()); + payload.setProperty("runtimePath", getStableAudioRuntimeRoot().getFullPathName()); + })); + + std::thread([this, sourcePath = sourceRoot.getFullPathName(), stableSessionId]() + { + const juce::File source(sourcePath); + const auto destination = getStableAudioModelRoot(); + const auto runtimeRoot = getStableAudioRuntimeRoot(); + const auto readyMarker = runtimeRoot.getChildFile(".openstudio-stable-audio-ready"); + juce::String error; + bool success = true; + auto progressForStableAudioCommand = [] (const juce::String& label) + { + if (label.containsIgnoreCase("Creating separate")) + return 0.25f; + if (label.containsIgnoreCase("Updating Stable Audio")) + return 0.35f; + if (label.containsIgnoreCase("runtime dependencies")) + return 0.50f; + if (label.containsIgnoreCase("Flash Attention")) + return 0.78f; + if (label.containsIgnoreCase("Validating")) + return 0.92f; + return 0.40f; + }; + auto runCommand = [&] (const juce::StringArray& command, + const juce::String& label, + int timeoutMs) -> bool + { + const auto commandStartedAt = juce::Time::getMillisecondCounterHiRes(); + const auto commandProgress = progressForStableAudioCommand(label); + auto lastUiUpdateMs = commandStartedAt; + juce::String commandOutput; + auto addOutputToStatus = [&] (const juce::String& outputChunk) + { + if (outputChunk.isEmpty()) + return; + + commandOutput += outputChunk; + juce::StringArray lines; + lines.addLines(outputChunk.replace("\r", "\n")); + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.progress = commandProgress; + status.elapsedMs = static_cast(juce::Time::getMillisecondCounterHiRes() - commandStartedAt); + for (const auto& rawLine : lines) + { + const auto trimmed = rawLine.trim(); + if (trimmed.isNotEmpty()) + status.activityLines.add(trimmed.substring(0, 900)); + } + }); + }; + + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("command", command.joinIntoString(" ")); + })); + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = commandProgress; + status.installInProgress = true; + status.message = label; + status.stepLabel = label; + status.lastPhase = "stable_audio_runtime"; + status.installSessionId = stableSessionId; + status.elapsedMs = 0; + const auto isLargeStableAudioStep = label.containsIgnoreCase("runtime dependencies") + || label.containsIgnoreCase("Flash Attention") + || label.containsIgnoreCase("PyTorch"); + status.downloadHint = isLargeStableAudioStep + ? "Stable Audio 3 dependencies can take several minutes to install. OpenStudio will keep updating this log while pip is running." + : juce::String(); + status.isLargeDownload = isLargeStableAudioStep; + status.activityLines.add(label); + }); + + auto process = std::make_shared(); + auto clearStableCommandProcess = [&] + { + const juce::ScopedLock lock(aiToolsStatusLock); + if (installProcess == process) + installProcess.reset(); + }; + { + const juce::ScopedLock lock(aiToolsStatusLock); + installProcess = process; + installCommandLine = command.joinIntoString(" "); + installSessionId = stableSessionId; + installLastObservedPhase = "stable_audio_runtime"; + } + + if (! process->start(command, juce::ChildProcess::wantStdOut | juce::ChildProcess::wantStdErr)) + { + clearStableCommandProcess(); + error = label + " could not be started."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("error", error); + })); + return false; + } + + for (;;) + { + addOutputToStatus(process->readAllProcessOutput()); + + if (! process->isRunning()) + break; + + if (aiToolsCancelRequested.load()) + { + process->kill(); + error = "Stable Audio 3 import was cancelled."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_cancelled", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + })); + clearStableCommandProcess(); + return false; + } + + const auto nowMs = juce::Time::getMillisecondCounterHiRes(); + if (nowMs - commandStartedAt >= static_cast(timeoutMs)) + { + process->kill(); + error = label + " timed out."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("error", error); + })); + clearStableCommandProcess(); + return false; + } + + if (nowMs - lastUiUpdateMs >= 2000.0) + { + lastUiUpdateMs = nowMs; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.state = "installing"; + status.progress = commandProgress; + status.installInProgress = true; + status.message = label; + status.stepLabel = label; + status.lastPhase = "stable_audio_runtime"; + status.elapsedMs = static_cast(nowMs - commandStartedAt); + status.activityLines.add(label + " still running..."); + }); + } + + juce::Thread::sleep(250); + } + + addOutputToStatus(process->readAllProcessOutput()); + if (aiToolsCancelRequested.load()) + { + clearStableCommandProcess(); + error = "Stable Audio 3 import was cancelled."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_cancelled", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + })); + return false; + } + const auto output = commandOutput.trim(); + const auto exitCode = process->getExitCode(); + clearStableCommandProcess(); + + if (exitCode != 0) + { + error = label + " failed."; + if (output.isNotEmpty()) + error += " " + output.substring(0, 900); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("exitCode", static_cast(exitCode)); + payload.setProperty("output", output.substring(0, 900)); + })); + return false; + } + + if (output.isNotEmpty()) + { + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + status.activityLines.add(output.substring(0, 900)); + }); + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_command_succeeded", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("label", label); + payload.setProperty("output", output.substring(0, 900)); + })); + return true; + }; + + if (aiToolsCancelRequested.load()) + { + success = false; + error = "Stable Audio 3 import was cancelled."; + } + else if (source.getFullPathName() != destination.getFullPathName()) + { + if (! destination.getParentDirectory().createDirectory()) + { + success = false; + error = "Could not create managed Stable Audio model directory."; + } + else + { + if (destination.exists()) + destination.deleteRecursively(); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + "stable_audio_copy_started", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + })); + success = source.copyDirectoryTo(destination); + if (! success) + error = "Could not copy Stable Audio 3 files into the managed OpenStudio model folder."; + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_copy_succeeded" : "stable_audio_copy_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + if (error.isNotEmpty()) + payload.setProperty("error", error); + })); + } + } + + if (success) + { + const auto missingAfterCopy = getMissingStableAudioFiles(destination); + if (! missingAfterCopy.isEmpty()) + { + success = false; + error = "Imported Stable Audio 3 snapshot is missing files: " + missingAfterCopy.joinIntoString(", "); + } + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_imported_layout_valid" : "stable_audio_imported_layout_invalid", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("destinationPath", destination.getFullPathName()); + juce::Array missing; + for (const auto& file : missingAfterCopy) + missing.add(file); + payload.setProperty("missingFiles", missing); + })); + } + + if (success && ! readyMarker.existsAsFile()) + { + if (! runtimeRoot.createDirectory()) + { + success = false; + error = "Could not create the Stable Audio 3 runtime directory."; + } + + auto runtimePython = findStableAudioPython(); + if (success && ! runtimePython.existsAsFile()) + { + auto bootstrapPython = findSystemPython(); + if (! bootstrapPython.existsAsFile()) + bootstrapPython = findPython(); + + if (! bootstrapPython.existsAsFile()) + { + success = false; + error = "Stable Audio 3 runtime setup needs Python, but no suitable Python executable was found."; + } + else + { + juce::StringArray command; + command.add(bootstrapPython.getFullPathName()); + command.add("-m"); + command.add("venv"); + command.add(runtimeRoot.getFullPathName()); + success = runCommand(command, "Creating separate Stable Audio 3 runtime...", 10 * 60 * 1000); + runtimePython = findStableAudioPython(); + if (success && ! runtimePython.existsAsFile()) + { + success = false; + error = "Stable Audio 3 runtime Python was not created."; + } + } + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("pip"); + success = runCommand(command, "Updating Stable Audio 3 runtime package installer...", 15 * 60 * 1000); + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("--force-reinstall"); + command.add("--index-url"); + command.add("https://download.pytorch.org/whl/cu128"); + command.add("torch==2.7.1"); + command.add("torchaudio==2.7.1"); + success = runCommand(command, "Installing Stable Audio 3 CUDA PyTorch runtime...", 60 * 60 * 1000); + if (! success) + error = "Could not install the Stable Audio 3 CUDA PyTorch runtime. " + error; + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-m"); + command.add("pip"); + command.add("install"); + command.add("--upgrade"); + command.add("git+https://github.com/Stability-AI/stable-audio-3.git"); + command.add("soundfile"); + success = runCommand(command, "Installing Stable Audio 3 runtime dependencies...", 60 * 60 * 1000); + } + + if (success) + { + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_runtime", + "stable_audio_flash_attention_skipped", + stableSessionId, + [] (juce::DynamicObject& payload) + { + payload.setProperty("reason", "flash-attn is optional for Stable Audio 3 on Windows; using PyTorch attention fallback"); + })); + } + + if (success) + { + juce::StringArray command; + command.add(runtimePython.getFullPathName()); + command.add("-c"); + command.add("import stable_audio_3, torch, torchaudio, soundfile; assert torch.cuda.is_available(), 'CUDA PyTorch is not available in the Stable Audio 3 runtime'; print('Stable Audio 3 runtime import check passed'); print('torch=' + torch.__version__ + ', cuda=' + str(torch.version.cuda))"); + success = runCommand(command, "Validating Stable Audio 3 runtime imports...", 5 * 60 * 1000); + } + + if (success && ! readyMarker.replaceWithText("ready\n", false, false, "\n")) + { + success = false; + error = "Could not write Stable Audio 3 runtime readiness marker."; + } + } + + aiToolsInstallWorkInProgress = false; + updateCachedAiToolsStatus([&] (AiToolsStatus& status) + { + const bool cancelled = aiToolsCancelRequested.load() + || error.containsIgnoreCase("cancelled"); + status.state = success ? "ready" : cancelled ? "cancelled" : "error"; + status.progress = success ? 1.0f : 0.0f; + status.installInProgress = false; + status.message = success + ? "Stable Audio 3 Medium is ready." + : error; + status.stepLabel = status.message; + status.lastPhase = "stable_audio_import"; + status.error = success ? juce::String() : error; + status.errorCode = success || cancelled ? juce::String() : "stable_audio_import_failed"; + status.terminalReason = status.errorCode; + status.activityLines.clear(); + status.activityLines.add(status.message); + status.detailLogPath = getAiToolsInstallLogFile().getFullPathName(); + }); + appendAiToolsLogLine(makeAiLogEvent("host", + "stable_audio_import", + success ? "stable_audio_import_finished" : "stable_audio_import_failed", + stableSessionId, + [&] (juce::DynamicObject& payload) + { + payload.setProperty("modelId", kStableAudioModelId); + payload.setProperty("sourcePath", source.getFullPathName()); + payload.setProperty("destinationPath", destination.getFullPathName()); + payload.setProperty("runtimePath", runtimeRoot.getFullPathName()); + if (error.isNotEmpty()) + payload.setProperty("error", error); + })); + }).detach(); + + result->setProperty("started", true); + result->setProperty("message", "Stable Audio 3 import started."); + result->setProperty("status", aiToolsStatusToVar(getCachedAiToolsStatusSnapshot())); + return juce::var(result.release()); + } + bool selectedFeaturesReady = true; for (const auto& feature : installOptions.selectedFeatures) { diff --git a/Source/StemSeparator.h b/Source/StemSeparator.h index 58fad0c..9a9d761 100644 --- a/Source/StemSeparator.h +++ b/Source/StemSeparator.h @@ -162,21 +162,33 @@ class StemSeparator /** Get the preferred stem-runtime root directory under user app-data. */ juce::File getUserRuntimeRoot() const; + /** Get the separate Stable Audio runtime root directory under user app-data. */ + juce::File getStableAudioRuntimeRoot() const; + /** Get the preferred models directory under user app-data. */ juce::File getUserModelsDir() const; /** Get the pinned ACE-Step checkpoint root used for music generation. */ juce::File getMusicGenerationCheckpointRoot() const; + /** Get the managed Stable Audio 3 Medium snapshot root. */ + juce::File getStableAudioModelRoot() const; + /** Find the prepared user-runtime Python executable. */ juce::File findPython() const; + /** Find the prepared Stable Audio runtime Python executable. */ + juce::File findStableAudioPython() const; + /** Find a user-managed Python interpreter suitable for bootstrapping installs. */ juce::File findSystemPython() const; /** Return true when this build is allowed to use external Python fallback. */ bool isExternalPythonFallbackEnabled() const; + /** Return true when the separate Stable Audio runtime is present. */ + bool hasStableAudioRuntime() const; + /** Resolve a Python executable from a runtime root. */ juce::File findPythonInRuntimeRoot (const juce::File& runtimeRoot) const; @@ -243,6 +255,9 @@ class StemSeparator bool userConfirmedDownload = false; juce::StringArray selectedFeatures; juce::String requestedFeature; + juce::String modelId; + juce::String stableAudioModelPath; + bool stableAudioLicenseAccepted = false; }; struct HardwareStatus @@ -280,6 +295,12 @@ class StemSeparator /** Return true if the preferred stem model is available. */ bool hasRequiredModel (const juce::File& modelsDir) const; + /** Return missing required files for a Stable Audio 3 Medium snapshot. */ + juce::StringArray getMissingStableAudioFiles (const juce::File& modelRoot) const; + + /** Return true if a Stable Audio 3 Medium snapshot contains the required files. */ + bool isStableAudioModelFolderValid (const juce::File& modelRoot) const; + /** Poll a running AI tools install and refresh the cached status. */ void pollInstallProgress(); @@ -333,7 +354,7 @@ class StemSeparator juce::String& errorCode) const; /** Serialize AI tools status to juce::var for the native bridge. */ - static juce::var aiToolsStatusToVar (const AiToolsStatus& status); + juce::var aiToolsStatusToVar (const AiToolsStatus& status) const; std::unique_ptr childProcess; std::shared_ptr installProcess; diff --git a/docs/stable_audio_3_integration_plan.md b/docs/stable_audio_3_integration_plan.md new file mode 100644 index 0000000..164cd30 --- /dev/null +++ b/docs/stable_audio_3_integration_plan.md @@ -0,0 +1,148 @@ +# Stable Audio 3 + ACE-Step Source-Audio Workflow Checklist + +## Summary + +- [x] Keep the current ACE model/version exactly as-is: `ACE-Step 1.5 XL Turbo`. +- [x] Add `Stable Audio 3 Medium` as a separate selectable model with its own runtime, setup, and params. +- [x] Add supported source-audio workflows for both models from the clip context menu. +- [x] Do not implement ACE Extract, Lego, or Complete: no UI entries, workflow IDs, backend branches, or disabled placeholders. + +## AI Track Flow + +- [x] Keep AI tracks prompt-first. +- [x] Add a compact model dropdown in the AI track header: + - `ACE-Step 1.5 XL Turbo` + - `Stable Audio 3 Medium` +- [x] Add the same model dropdown at the top of the AI track params modal. +- [x] Make model changes update the workflow dropdown and visible params form. +- [x] Limit AI track workflows to: + - ACE-Step: `Text to Music`, `Lyrics + Style` + - Stable Audio 3: `Text to Audio` +- [x] If the selected model is unavailable, turn Generate into a setup CTA focused on that model. + +## Clip Context Menu Flow + +- [x] Add an `AI Generation` submenu to the existing audio clip right-click menu, near `Separate Stems...`. +- [x] Show the submenu only for audio clips. +- [x] Add menu items: + - `Create Variation...` + - `Inpaint Selection...` + - `Continue Clip...` +- [x] Disable `Inpaint Selection...` unless the current time selection overlaps the clicked clip. +- [x] Open a new `AI Clip Generation` modal from each menu item. +- [x] Include source clip name, source track name, and duration/range summary in the modal. +- [x] Limit modal model choices to models that support the selected source workflow. +- [x] Include workflow-specific params, setup/status block, and Generate/Cancel footer. +- [x] Do not show ACE Extract/Lego/Complete anywhere. + +## Output Placement + +- [x] Make source-audio workflows nondestructive by default. +- [x] `Create Variation...`: create a new audio track directly below the source track named `AI Variation - `, aligned to the source clip start. +- [x] `Inpaint Selection...`: create a new audio track directly below the source track named `AI Inpaint - `, aligned to the source clip start. +- [x] `Continue Clip...`: insert only the generated tail at `sourceClip.startTime + sourceClip.duration`. +- [x] Place continuation on the source track when clear; otherwise create `AI Continuation - ` below the source track. +- [x] Make all generated track/clip additions undo-tracked. + +## Workflow + Params Registry + +- [x] Replace the single ACE-only workflow registry with a model-aware registry. +- [x] Add model IDs: + - `ace-step-v15-xl-turbo` + - `stable-audio-3-medium` +- [x] Add workflow IDs: + - `text-to-music` + - `lyrics-style` + - `text-to-audio` + - `variation` + - `inpaint-selection` + - `continue-clip` +- [x] Declare supported model IDs, surface, params schema, and source requirements per workflow. +- [x] Add ACE source params: prompt, lyrics, seed, duration/extension duration, inference steps, guidance/cfg, `audio_cover_strength`, repaint start/end. +- [x] Add Stable Audio params: prompt, negative prompt, seed, duration/extension duration, steps, cfg scale, source strength/noise amount, inpaint range. +- [x] Add Stable Audio Advanced LoRA inference fields for optional `.safetensors` path and strength. + +## Frontend State + UI + +- [x] Add `aiMusicModelId` to AI tracks and default existing projects to ACE-Step. +- [x] Add modal state for clip generation source, workflow, model, params, and validation/status. +- [x] Build `AIClipGenerationModal` as a sibling to `StemSeparationModal`. +- [x] Reuse current dark theme tokens, compact controls, existing modal/footer layout, and existing UI components. +- [x] Convert time selection to clip-relative inpaint range: + - `rangeStart = max(timeSelection.start, clip.startTime) - clip.startTime` + - `rangeEnd = min(timeSelection.end, clipEnd) - clip.startTime` + - disable when `rangeEnd <= rangeStart` + +## Backend + Bridge + +- [x] Extend generation start API to `startAIGeneration(trackId, modelId, workflowId, paramsJSON)`. +- [x] Preserve old three-argument behavior by treating it as ACE-Step. +- [x] Enforce one global generation job at a time for v1. +- [x] Refactor `AITrackEngine` into provider routing. +- [x] Keep ACE provider on the existing persistent ACE worker. +- [x] Add a separate Stable Audio provider/runtime. +- [x] Include source file path, clip offset, clip duration, source track/clip IDs, inpaint range, or extension length in source params. +- [x] Prepare exact source-segment temp WAV before inference so trimmed clips behave correctly. +- [x] Map ACE Variation to Cover with `src_audio` and `audio_cover_strength`. +- [x] Map ACE Inpaint to Repaint with `src_audio`, `repainting_start`, and `repainting_end`. +- [x] Map ACE Continue to Repaint at the source clip end and crop the output to the generated tail before import. +- [x] Map Stable Audio Variation, Inpaint, and Continue to its source-conditioned workflows. +- [x] Add optional `modelId`, `workflowId`, and `sourceClipId` to progress payloads. + +## Setup Flow + +- [x] Add a music model section under AI Tools Setup > Audio Generation. +- [x] Keep ACE setup unchanged. +- [x] Add Stable Audio 3 Medium setup as strict opt-in. +- [x] Show Stability/Gemma license notice and require explicit checkbox confirmation. +- [x] Show `Powered by Stability AI` attribution when Stable Audio is selected or used. +- [x] Use manual Hugging Face download/import, not stored HF credentials. +- [x] Add `Open Hugging Face Model Page`. +- [x] Show required folder layout help text. +- [x] Add `Proceed with Setup` folder picker and validation. +- [x] Validate Stable Audio folder files: + - `model.safetensors` + - `model_config.json` + - `LICENSE.md` + - `LICENSE_GEMMA.md` + - `NOTICE` + - `t5gemma-b-b-ul2/model.safetensors` + - `t5gemma-b-b-ul2/config.json` + - `t5gemma-b-b-ul2/tokenizer.json` + - `t5gemma-b-b-ul2/tokenizer.model` + - `t5gemma-b-b-ul2/tokenizer_config.json` + - `t5gemma-b-b-ul2/special_tokens_map.json` +- [x] Validate the initial local target `C:\Users\srvds\Downloads\stable_audio_3`. +- [x] Copy/import the snapshot into a managed OpenStudio model folder. +- [x] Keep Stable Audio runtime separate from ACE. + +## Test + Visual Harness + +- [x] Add unit tests for model-aware workflow filtering, default params, normalization, inpaint range conversion, disabled inpaint, and no unsupported workflow IDs. +- [x] Add store tests for model changes, undoable generated source outputs, and continuation placement. +- [x] Add installer/probe tests for Stable Audio folder validation and license confirmation. +- [x] Add `tools/ai-generation-ui-harness.mjs`. +- [x] Harness scenario: `ai-track-model-selector`. +- [x] Harness scenario: `clip-context-ai-menu`. +- [x] Harness scenario: `ai-clip-generation-modal`. +- [x] Harness scenario: `stable-audio-setup`. +- [x] Harness checks desktop and compact screenshots. +- [x] Harness checks text overflow, viewport bounds, menu placement, and theme alignment. +- [x] Harness writes `qa/ai-generation//screenshots` and `qa/ai-generation//report.json`. + +## Verification + +- [x] Run `npx tsc --noEmit`. +- [x] Run the frontend visual harness. +- [x] Build frontend assets into `frontend/dist`. +- [x] Run `cmake --build build --config Debug`. +- [x] Stop Codex-started dev servers and leave port `5173` free. +- [ ] Manual check: ACE text generation from AI track. +- [ ] Manual check: ACE lyrics generation from AI track. +- [ ] Manual check: ACE clip variation. +- [ ] Manual check: ACE inpaint with overlapping time selection. +- [ ] Manual check: ACE continuation. +- [ ] Manual check: Stable Audio setup using `C:\Users\srvds\Downloads\stable_audio_3`. +- [ ] Manual check: Stable Audio text generation from AI track. +- [ ] Manual check: Stable Audio variation, inpaint, and continuation. +- [ ] Verify generated WAVs import, appear at expected positions, play back, and can be undone. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7f5fc2e..9ed2ef9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -82,6 +82,7 @@ const TimecodeSettingsPanel = React.lazy(() => import("./components/TimecodeSett const HelpOverlay = React.lazy(() => import("./components/HelpOverlay").then(m => ({ default: m.HelpOverlay }))); const GettingStartedGuide = React.lazy(() => import("./components/GettingStartedGuide").then(m => ({ default: m.GettingStartedGuide }))); const StemSeparationModal = React.lazy(() => import("./components/StemSeparationModal")); +const AIClipGenerationModal = React.lazy(() => import("./components/AIClipGenerationModal")); const AiToolsSetupModal = React.lazy(() => import("./components/AiToolsSetupModal")); import { DndContext, @@ -171,6 +172,7 @@ function App() { masterAutomationLanes, showMasterAutomation, showStemSeparation, + showAIClipGeneration, showAiToolsSetup, closeAiToolsSetup, hydrateRecentProjects, @@ -246,6 +248,7 @@ function App() { masterAutomationLanes: state.masterAutomationLanes, showMasterAutomation: state.showMasterAutomation, showStemSeparation: state.showStemSeparation, + showAIClipGeneration: state.showAIClipGeneration, showAiToolsSetup: state.showAiToolsSetup, closeAiToolsSetup: state.closeAiToolsSetup, hydrateRecentProjects: state.hydrateRecentProjects, @@ -395,7 +398,6 @@ function App() { const previousAiToolsFullReadyRef = useRef(Boolean( aiToolsStatus.musicGenerationReady && aiToolsStatus.musicGenerationLayoutValid - && (aiToolsStatus.musicGenerationPerformanceReady ?? true) && hasNativeMusicProfile, )); const [aiToolsInstallStatusStale, setAiToolsInstallStatusStale] = useState(false); @@ -475,7 +477,6 @@ function App() { currentState === "ready" && aiToolsStatus.musicGenerationReady && aiToolsStatus.musicGenerationLayoutValid - && (aiToolsStatus.musicGenerationPerformanceReady ?? true) && hasNativeMusicProfile, ); const currentPartialReady = Boolean( @@ -483,7 +484,6 @@ function App() { && ( !aiToolsStatus.musicGenerationReady || !aiToolsStatus.musicGenerationLayoutValid - || !(aiToolsStatus.musicGenerationPerformanceReady ?? true) || !hasNativeMusicProfile ), ); @@ -1943,6 +1943,13 @@ function App() { )} + {/* AI Clip Generation Modal */} + {showAIClipGeneration && ( + + + + )} + {/* Channel Strip EQ Modal */} {showChannelStripEQ && ( @@ -2041,6 +2048,7 @@ function App() { {/* AI Tools Background Install Popup */} {!showStemSeparation && + !showAIClipGeneration && !showAiToolsSetup && showAiToolsBackgroundInstallPopup && aiToolsStatus.installInProgress && ( diff --git a/frontend/src/__tests__/aiClipGenerationModal.test.ts b/frontend/src/__tests__/aiClipGenerationModal.test.ts new file mode 100644 index 0000000..fed31c1 --- /dev/null +++ b/frontend/src/__tests__/aiClipGenerationModal.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import modalSource from "../components/AIClipGenerationModal.tsx?raw"; + +describe("AI clip generation modal", () => { + it("requires direction prompts for Stable Audio source workflows", () => { + expect(modalSource).toContain("stableSourcePromptMissing"); + expect(modalSource).toContain("Stable Audio source workflows need a direction prompt."); + expect(modalSource).toContain("disabled={!sourceClip || (isModelReady && (inpaintSelectionMissing || stableSourcePromptMissing))}"); + }); +}); diff --git a/frontend/src/__tests__/aiGenerationStore.test.ts b/frontend/src/__tests__/aiGenerationStore.test.ts new file mode 100644 index 0000000..f34ef62 --- /dev/null +++ b/frontend/src/__tests__/aiGenerationStore.test.ts @@ -0,0 +1,130 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { STABLE_AUDIO_3_MODEL_ID } from "../data/aiWorkflows"; +import { nativeBridge } from "../services/NativeBridge"; +import { commandManager } from "../store/commands"; +import { createDefaultTrack, type AudioClip, useDAWStore } from "../store/useDAWStore"; + +const initialState = useDAWStore.getState(); + +function sourceClip(overrides: Partial = {}): AudioClip { + return { + id: "clip-source", + filePath: "C:/audio/source.wav", + name: "Source Loop", + startTime: 10, + duration: 5, + offset: 0, + color: "#4cc9f0", + volumeDB: 0, + fadeIn: 0, + fadeOut: 0, + sampleRate: 44100, + sourceLength: 5, + ...overrides, + }; +} + +describe("AI generation store actions", () => { + beforeEach(() => { + commandManager.clear(); + useDAWStore.setState(initialState); + vi.spyOn(nativeBridge, "importMediaFile").mockResolvedValue({ + filePath: "C:/audio/generated.wav", + duration: 3, + sampleRate: 44100, + numChannels: 2, + format: "wav", + }); + vi.spyOn(nativeBridge, "addTrack").mockResolvedValue("track-added"); + vi.spyOn(nativeBridge, "removeTrack").mockResolvedValue(true); + vi.spyOn(nativeBridge, "addPlaybackClip").mockResolvedValue(true); + vi.spyOn(nativeBridge, "removePlaybackClip").mockResolvedValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + commandManager.clear(); + useDAWStore.setState(initialState); + }); + + it("resets incompatible AI track workflow params when the model changes", () => { + const aiTrack = createDefaultTrack("ai-track", "AI Track", "#7c3aed", "ai"); + useDAWStore.setState({ tracks: [aiTrack], canUndo: false, canRedo: false }); + + useDAWStore.getState().setAITrackModel("ai-track", STABLE_AUDIO_3_MODEL_ID); + + const updated = useDAWStore.getState().tracks[0]; + expect(updated.aiMusicModelId).toBe(STABLE_AUDIO_3_MODEL_ID); + expect(updated.aiWorkflow).toBe("text-to-audio"); + expect(updated.aiWorkflowParams).toMatchObject({ + prompt: "", + negative_prompt: "", + duration: 30, + }); + expect(useDAWStore.getState().canUndo).toBe(true); + + useDAWStore.getState().undo(); + expect(useDAWStore.getState().tracks[0].aiWorkflow).toBe("text-to-music"); + }); + + it("adds variation output as an undoable track below the source track", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "variation", + filePath: "C:/audio/generated.wav", + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(2); + expect(tracks[1].name).toBe("AI Variation - Source Loop"); + expect(tracks[1].clips[0].startTime).toBe(10); + expect(useDAWStore.getState().canUndo).toBe(true); + + useDAWStore.getState().undo(); + expect(useDAWStore.getState().tracks).toHaveLength(1); + }); + + it("places continuation on the source track when the tail range is clear", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [sourceClip()]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "continue-clip", + filePath: "C:/audio/generated.wav", + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(1); + expect(tracks[0].clips).toHaveLength(2); + expect(tracks[0].clips[1].startTime).toBe(15); + }); + + it("places continuation on a new track when the source tail range collides", async () => { + const track = createDefaultTrack("track-source", "Guitar", "#4cc9f0", "audio"); + track.clips = [ + sourceClip(), + sourceClip({ id: "clip-existing", name: "Existing", startTime: 16, duration: 2 }), + ]; + useDAWStore.setState({ tracks: [track], canUndo: false, canRedo: false }); + + await useDAWStore.getState().addGeneratedSourceAudioClip({ + sourceTrackId: "track-source", + sourceClipId: "clip-source", + workflowId: "continue-clip", + filePath: "C:/audio/generated.wav", + }); + + const tracks = useDAWStore.getState().tracks; + expect(tracks).toHaveLength(2); + expect(tracks[1].name).toBe("AI Continuation - Source Loop"); + expect(tracks[1].clips[0].startTime).toBe(15); + }); +}); diff --git a/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts b/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts index eb29985..ce7521d 100644 --- a/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts +++ b/frontend/src/__tests__/aiToolsFeatureInstaller.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import modalSource from "../components/AiToolsSetupModal.tsx?raw"; import bridgeSource from "../services/NativeBridge.ts?raw"; import storeSource from "../store/useDAWStore.ts?raw"; +import mainComponentSource from "../../../Source/MainComponent.cpp?raw"; +import stemSeparatorSource from "../../../Source/StemSeparator.cpp?raw"; describe("AI feature installer contract", () => { it("defaults the setup checklist to requested compatible features or stem separation", () => { @@ -12,7 +14,10 @@ describe("AI feature installer contract", () => { }); it("keeps incompatible Audio Generation disabled while Stem Separation remains selectable", () => { - expect(modalSource).toContain("!feature.compatible"); + expect(modalSource).toContain("function buildSetupCatalog"); + expect(modalSource).toContain("disabled = !item.compatible && !item.ready"); + expect(modalSource).toContain("title={disabled ? item.disabledReason : statusLabel(item)}"); + expect(modalSource).toContain("Hardware not supported"); expect(modalSource).toContain("Incompatible features are disabled and will not be installed."); expect(modalSource).toContain("supported GPU with at least 8 GB memory was not detected"); expect(modalSource).toContain("CPU-only machines are supported."); @@ -29,4 +34,55 @@ describe("AI feature installer contract", () => { expect(storeSource).toContain("options.requestedFeature ?? \"stemSeparation\""); expect(storeSource).toContain("selectedFeatures: [\"stemSeparation\"]"); }); + + it("exposes strict Stable Audio 3 manual import controls", () => { + expect(modalSource).toContain("STABLE_AUDIO_MODEL_URL"); + expect(modalSource).toContain("Open Hugging Face Model Page"); + expect(modalSource).toContain("Proceed with Setup"); + expect(modalSource).toContain("Use Downloads Folder"); + expect(modalSource).toContain("Cancel Setup"); + expect(modalSource).toContain("stableAudioSelectedFolder"); + expect(modalSource).toContain("modelId: STABLE_AUDIO_3_MODEL_ID"); + expect(modalSource).toContain("stableAudioLicenseAccepted"); + expect(modalSource).toContain("LICENSE_GEMMA.md"); + expect(bridgeSource).toContain("stableAudioModelPath?: string"); + expect(bridgeSource).toContain("stableAudioLicenseAccepted?: boolean"); + expect(bridgeSource).toContain("modelId?: \"ace-step-v15-xl-turbo\" | \"stable-audio-3-medium\""); + expect(mainComponentSource).toContain(".withNativeFunction (\"browseForFolder\""); + expect(mainComponentSource).toContain("canSelectDirectories"); + }); + + it("validates Stable Audio 3 folder layout and license before import", () => { + expect(stemSeparatorSource).toContain("model.safetensors"); + expect(stemSeparatorSource).toContain("t5gemma-b-b-ul2/tokenizer.model"); + expect(stemSeparatorSource).toContain("Stable Audio 3 setup requires accepting"); + expect(stemSeparatorSource).toContain("stable_audio_model_path_required"); + expect(stemSeparatorSource).toContain("stable_audio_model_layout_invalid"); + expect(stemSeparatorSource).toContain("stable_audio_import_requested"); + expect(stemSeparatorSource).toContain("stable_audio_command_started"); + expect(stemSeparatorSource).toContain("Stable Audio 3 dependencies can take several minutes to install."); + expect(stemSeparatorSource).toContain("stable_audio_command_cancelled"); + expect(stemSeparatorSource).toContain("copyDirectoryTo(destination)"); + }); + + it("keeps Stable Audio runtime separate from the ACE runtime", () => { + expect(stemSeparatorSource).toContain("getStableAudioRuntimeRoot"); + expect(stemSeparatorSource).toContain("stable-audio-runtime"); + expect(stemSeparatorSource).toContain("Installing Stable Audio 3 CUDA PyTorch runtime"); + expect(stemSeparatorSource).toContain("git+https://github.com/Stability-AI/stable-audio-3.git"); + expect(stemSeparatorSource).toContain("stable_audio_flash_attention_skipped"); + expect(stemSeparatorSource).toContain("PyTorch attention fallback"); + }); + + it("keeps Stable Audio install options intact through the store action", () => { + expect(storeSource).toContain("const isStableAudioImport = options.modelId === STABLE_AUDIO_3_MODEL_ID"); + expect(storeSource).toContain("...options"); + expect(storeSource).toContain("Stable Audio 3 setup is starting in the background."); + }); + + it("sanitizes internal ACE runtime paths from setup UI copy", () => { + expect(modalSource).toContain("function sanitizeSetupMessage"); + expect(modalSource).toContain("ACE-Step runtime files are missing. Repair or reinstall ACE-Step Audio Generation setup."); + expect(modalSource).toContain("displayActivityLines"); + }); }); diff --git a/frontend/src/__tests__/aiWorkflowParams.test.ts b/frontend/src/__tests__/aiWorkflowParams.test.ts index 3b0e1e1..9e62022 100644 --- a/frontend/src/__tests__/aiWorkflowParams.test.ts +++ b/frontend/src/__tests__/aiWorkflowParams.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; import { + ACE_STEP_MODEL_ID, + AI_WORKFLOWS, + STABLE_AUDIO_3_MODEL_ID, + getAIModelsForWorkflow, + getAIWorkflowsForSurface, + getClipInpaintRange, getDefaultWorkflowParams, normalizeWorkflowParams, } from "../data/aiWorkflows"; @@ -100,4 +106,117 @@ describe("AI workflow params", () => { generate_audio_codes: true, }); }); + + it("filters workflows by surface and selected model", () => { + expect(getAIWorkflowsForSurface("ai-track", ACE_STEP_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "text-to-music", + "lyrics-style", + ]); + expect(getAIWorkflowsForSurface("ai-track", STABLE_AUDIO_3_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "text-to-audio", + ]); + expect(getAIWorkflowsForSurface("clip-context", STABLE_AUDIO_3_MODEL_ID).map((workflow) => workflow.id)).toEqual([ + "variation", + "inpaint-selection", + "continue-clip", + ]); + }); + + it("uses Stable Audio defaults and clamps source workflow params", () => { + const defaults = getDefaultWorkflowParams("text-to-audio", STABLE_AUDIO_3_MODEL_ID); + expect(defaults).toMatchObject({ + prompt: "", + negative_prompt: "", + seed: -1, + duration: 30, + steps: 8, + cfg_scale: 1, + lora_path: "", + lora_strength: 1, + }); + + const normalized = normalizeWorkflowParams("variation", { + steps: "999", + cfg_scale: "-4", + noise_amount: "-1", + lora_strength: "5", + }, STABLE_AUDIO_3_MODEL_ID); + + expect(normalized).toMatchObject({ + steps: 32, + cfg_scale: 0.1, + noise_amount: 0, + lora_strength: 2, + }); + }); + + it("keeps clip source workflow forms source-specific", () => { + const stableVariation = getDefaultWorkflowParams("variation", STABLE_AUDIO_3_MODEL_ID); + expect(stableVariation).toMatchObject({ + prompt: "Create a close musical variation that preserves the source clip's tempo, key, primary instrument, arrangement density, and mix character. Do not add vocals or unrelated instruments unless requested.", + negative_prompt: "unrelated instruments, unexpected vocals, full band arrangement unless present in the source, distorted, noisy, clipped, abrupt transition, low quality", + seed: -1, + noise_amount: 0.5, + steps: 8, + cfg_scale: 1, + }); + expect(stableVariation).not.toHaveProperty("duration"); + expect(stableVariation).not.toHaveProperty("extension_duration"); + expect(stableVariation).not.toHaveProperty("inpaint_start"); + + const stableContinue = getDefaultWorkflowParams("continue-clip", STABLE_AUDIO_3_MODEL_ID); + expect(stableContinue).toMatchObject({ + prompt: "Continue the same musical idea, matching the source clip's tempo, key, primary instrument, harmony, room tone, and mix. Do not add vocals or unrelated instruments unless requested.", + negative_prompt: "unrelated instruments, unexpected vocals, full band arrangement unless present in the source, distorted, noisy, clipped, abrupt transition, low quality", + seed: -1, + extension_duration: 8, + steps: 8, + cfg_scale: 1, + }); + expect(stableContinue).not.toHaveProperty("duration"); + expect(stableContinue).not.toHaveProperty("noise_amount"); + + const stableInpaint = getDefaultWorkflowParams("inpaint-selection", STABLE_AUDIO_3_MODEL_ID); + expect(stableInpaint.prompt).toBe("Replace the selected range naturally while matching the surrounding clip."); + expect(stableInpaint).not.toHaveProperty("duration"); + + const aceVariation = getDefaultWorkflowParams("variation", ACE_STEP_MODEL_ID); + expect(aceVariation).toMatchObject({ + prompt: "Create a close musical variation that preserves the source clip's tempo, key, primary instrument, and arrangement density. Do not add vocals or unrelated instruments unless requested.", + seed: -1, + audio_cover_strength: 0.55, + generate_audio_codes: false, + inferenceSteps: 8, + }); + expect(aceVariation).not.toHaveProperty("lyrics"); + expect(aceVariation).not.toHaveProperty("duration"); + expect(aceVariation).not.toHaveProperty("extension_duration"); + }); + + it("converts timeline time selection into clip-relative inpaint range", () => { + expect(getClipInpaintRange({ start: 12, end: 16 }, { startTime: 10, duration: 8 })).toEqual({ + start: 2, + end: 6, + }); + expect(getClipInpaintRange({ start: 1, end: 4 }, { startTime: 10, duration: 8 })).toBeNull(); + expect(getClipInpaintRange(null, { startTime: 10, duration: 8 })).toBeNull(); + }); + + it("does not expose unsupported ACE-only workflow ids", () => { + const ids = AI_WORKFLOWS.map((workflow) => workflow.id); + expect(ids).not.toContain("extract"); + expect(ids).not.toContain("lego"); + expect(ids).not.toContain("complete"); + expect(ids).not.toContain("continuation"); + }); + + it("limits source workflow model choices to supporting models", () => { + expect(getAIModelsForWorkflow("variation").map((model) => model.id)).toEqual([ + ACE_STEP_MODEL_ID, + STABLE_AUDIO_3_MODEL_ID, + ]); + expect(getAIModelsForWorkflow("text-to-audio").map((model) => model.id)).toEqual([ + STABLE_AUDIO_3_MODEL_ID, + ]); + }); }); diff --git a/frontend/src/__tests__/interactionSafetyGuards.test.ts b/frontend/src/__tests__/interactionSafetyGuards.test.ts index 0fe3556..f2f50bd 100644 --- a/frontend/src/__tests__/interactionSafetyGuards.test.ts +++ b/frontend/src/__tests__/interactionSafetyGuards.test.ts @@ -45,6 +45,13 @@ describe("interaction safety guards", () => { expect(midiWindowSource).toContain("installModalContextMenuLeakGuard()"); }); + it("keeps context submenus open while crossing into the submenu", () => { + expect(contextMenuSource).toContain("submenuCloseTimerRef"); + expect(contextMenuSource).toContain("scheduleSubmenuClose"); + expect(contextMenuSource).toContain("onMouseEnter={clearSubmenuCloseTimer}"); + expect(contextMenuSource).toContain("left-full top-0 -ml-px"); + }); + it("keeps piano roll resize and modal context-menu paths guarded", () => { expect(pianoRollSource).toContain("Math.max(1, containerRef.current.clientWidth)"); expect(pianoRollSource).toContain("Math.max(1, containerRef.current.clientHeight)"); diff --git a/frontend/src/__tests__/timelineDragAxisLock.test.ts b/frontend/src/__tests__/timelineDragAxisLock.test.ts new file mode 100644 index 0000000..4c47020 --- /dev/null +++ b/frontend/src/__tests__/timelineDragAxisLock.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + getTimelineAxisLockedDeltas, + resolveTimelineDragAxisLock, +} from "../utils/timelineDragAxisLock"; + +describe("timeline drag axis lock", () => { + it("does not lock or move before the drag threshold", () => { + expect(resolveTimelineDragAxisLock(true, null, 3, 5)).toBeNull(); + expect(getTimelineAxisLockedDeltas(true, null, 3, 5)).toEqual({ + axisLock: null, + deltaX: 0, + deltaY: 0, + }); + }); + + it("locks horizontally when horizontal movement dominates", () => { + expect(resolveTimelineDragAxisLock(true, null, 12, 4)).toBe("x"); + expect(getTimelineAxisLockedDeltas(true, null, 12, 4)).toEqual({ + axisLock: "x", + deltaX: 12, + deltaY: 0, + }); + }); + + it("locks vertically when vertical movement dominates", () => { + expect(resolveTimelineDragAxisLock(true, null, 4, 12)).toBe("y"); + expect(getTimelineAxisLockedDeltas(true, null, 4, 12)).toEqual({ + axisLock: "y", + deltaX: 0, + deltaY: 12, + }); + }); + + it("chooses horizontal lock on ties", () => { + expect(resolveTimelineDragAxisLock(true, null, 8, -8)).toBe("x"); + }); + + it("keeps an existing lock for the rest of the drag", () => { + expect(resolveTimelineDragAxisLock(true, "y", 30, 1)).toBe("y"); + expect(getTimelineAxisLockedDeltas(true, "y", 30, 1)).toEqual({ + axisLock: "y", + deltaX: 0, + deltaY: 1, + }); + }); + + it("does not constrain deltas when Shift axis lock was not requested", () => { + expect(getTimelineAxisLockedDeltas(false, null, 4, 12)).toEqual({ + axisLock: null, + deltaX: 4, + deltaY: 12, + }); + }); +}); diff --git a/frontend/src/components/AIClipGenerationModal.tsx b/frontend/src/components/AIClipGenerationModal.tsx new file mode 100644 index 0000000..bde6f31 --- /dev/null +++ b/frontend/src/components/AIClipGenerationModal.tsx @@ -0,0 +1,813 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { AlertTriangle, CheckCircle2, ChevronDown, Music2, Settings2, WandSparkles } from "lucide-react"; +import { useShallow } from "zustand/shallow"; +import { + AI_WORKFLOW_SECTION_LABELS, + STABLE_AUDIO_3_MODEL_ID, + type AIWorkflowParam, + type AIWorkflowSection, + getAIModelsForWorkflow, + getAIWorkflow, + getAiMusicModel, + getClipInpaintRange, + mergeWorkflowParams, + normalizeWorkflowParams, + resolveAiMusicModelId, +} from "../data/aiWorkflows"; +import { nativeBridge, type AIGenerationProgress } from "../services/NativeBridge"; +import { type AudioClip, useDAWStore } from "../store/useDAWStore"; +import { + Button, + Checkbox, + Input, + Modal, + ModalContent, + ModalFooter, + ModalHeader, + Select, + Slider, + Textarea, +} from "./ui"; + +const SECTION_ORDER: AIWorkflowSection[] = [ + "prompt", + "source", + "sampling", + "generation", + "advanced", +]; + +const ADVANCED_SECTIONS = new Set([ + "sampling", + "generation", + "advanced", +]); + +const RANGE_PARAM_KEYS = new Set([ + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", +]); + +const STABLE_SOURCE_WORKFLOW_IDS = new Set([ + "variation", + "inpaint-selection", + "continue-clip", +]); + +function formatDuration(seconds: number) { + const safeSeconds = Math.max(0, seconds || 0); + const minutes = Math.floor(safeSeconds / 60); + const wholeSeconds = Math.floor(safeSeconds % 60); + return `${minutes}:${wholeSeconds.toString().padStart(2, "0")}`; +} + +function formatRange(range: { start: number; end: number } | null) { + if (!range) return "No overlapping time selection"; + return `${range.start.toFixed(2)}s - ${range.end.toFixed(2)}s`; +} + +function formatPhaseLabel(phase?: string) { + if (!phase) return "Preparing"; + return phase + .split("_") + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function formatElapsedLabel(elapsedMs?: number) { + if (!elapsedMs || elapsedMs <= 0) return ""; + const totalSeconds = Math.max(0, Math.floor(elapsedMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s elapsed` : `${seconds}s elapsed`; +} + +function formatEtaLabel(etaMs?: number) { + if (!etaMs || etaMs <= 0) return ""; + const totalSeconds = Math.max(0, Math.floor(etaMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return minutes > 0 ? `${minutes}m ${seconds}s left` : `${seconds}s left`; +} + +function formatRuntimeProfileLabel(profile?: string) { + switch (profile) { + case "native-xl-turbo": + case "openstudio-ace-split": + return "OpenStudio ACE Split"; + default: + return profile ? formatPhaseLabel(profile) : ""; + } +} + +function formatSessionModeLabel(sessionMode?: string) { + switch (sessionMode) { + case "persistent": + return "Persistent session"; + case "oneshot-fallback": + return "One-shot fallback"; + case "oneshot": + return "One-shot"; + default: + return sessionMode ? formatPhaseLabel(sessionMode) : ""; + } +} + +function progressWidth(progress?: number) { + return `${Math.max(4, Math.round((progress ?? 0) * 100))}%`; +} + +function shouldShowParam(workflowId: string, param: AIWorkflowParam) { + if (workflowId === "variation") { + return ![ + "extension_duration", + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", + "generate_audio_codes", + ].includes(param.key); + } + + if (workflowId === "inpaint-selection") { + return !["extension_duration", "generate_audio_codes"].includes(param.key); + } + + if (workflowId === "continue-clip") { + return ![ + "duration", + "repainting_start", + "repainting_end", + "inpaint_start", + "inpaint_end", + "generate_audio_codes", + ].includes(param.key); + } + + return true; +} + +export default function AIClipGenerationModal() { + const { + showAIClipGeneration, + aiClipGenerationTrackId, + aiClipGenerationClipId, + aiClipGenerationWorkflowId, + aiClipGenerationModelId, + aiClipGenerationParams, + aiClipGenerationRange, + aiClipGenerationError, + tracks, + aiToolsStatus, + closeAIClipGeneration, + setAIClipGenerationModel, + setAIClipGenerationParams, + setAIClipGenerationRange, + setAIClipGenerationError, + addGeneratedSourceAudioClip, + openAiToolsSetup, + } = useDAWStore( + useShallow((state) => ({ + showAIClipGeneration: state.showAIClipGeneration, + aiClipGenerationTrackId: state.aiClipGenerationTrackId, + aiClipGenerationClipId: state.aiClipGenerationClipId, + aiClipGenerationWorkflowId: state.aiClipGenerationWorkflowId, + aiClipGenerationModelId: state.aiClipGenerationModelId, + aiClipGenerationParams: state.aiClipGenerationParams, + aiClipGenerationRange: state.aiClipGenerationRange, + aiClipGenerationError: state.aiClipGenerationError, + tracks: state.tracks, + aiToolsStatus: state.aiToolsStatus, + closeAIClipGeneration: state.closeAIClipGeneration, + setAIClipGenerationModel: state.setAIClipGenerationModel, + setAIClipGenerationParams: state.setAIClipGenerationParams, + setAIClipGenerationRange: state.setAIClipGenerationRange, + setAIClipGenerationError: state.setAIClipGenerationError, + addGeneratedSourceAudioClip: state.addGeneratedSourceAudioClip, + openAiToolsSetup: state.openAiToolsSetup, + })), + ); + + const [isGenerating, setIsGenerating] = useState(false); + const [advancedOpen, setAdvancedOpen] = useState(false); + const [detailsOpen, setDetailsOpen] = useState(false); + const [progress, setProgress] = useState({ state: "idle", progress: 0 }); + const pollRef = useRef | null>(null); + const completedRef = useRef(false); + + const sourceTrack = useMemo( + () => tracks.find((track) => track.id === aiClipGenerationTrackId), + [aiClipGenerationTrackId, tracks], + ); + const sourceClip = useMemo( + () => sourceTrack?.clips.find((clip) => clip.id === aiClipGenerationClipId), + [aiClipGenerationClipId, sourceTrack], + ); + const workflow = getAIWorkflow( + aiClipGenerationWorkflowId, + aiClipGenerationModelId, + "clip-context", + ); + const model = getAiMusicModel(aiClipGenerationModelId); + const supportedModels = getAIModelsForWorkflow(workflow.id); + const params = useMemo( + () => mergeWorkflowParams(workflow.id, aiClipGenerationParams, aiClipGenerationModelId), + [aiClipGenerationModelId, aiClipGenerationParams, workflow.id], + ); + const selectedModelStatus = aiToolsStatus.musicModels?.[aiClipGenerationModelId]; + const isModelReady = Boolean( + selectedModelStatus?.ready + ?? ( + aiClipGenerationModelId === "ace-step-v15-xl-turbo" + ? ( + aiToolsStatus.features?.audioGeneration?.ready + ?? ( + aiToolsStatus.musicGenerationReady + && aiToolsStatus.musicGenerationLayoutValid + ) + ) + : false + ), + ); + const modelBlockedMessage = + selectedModelStatus?.message + || selectedModelStatus?.blockReason + || aiToolsStatus.features?.audioGeneration?.message + || aiToolsStatus.musicGenerationStatusMessage + || aiToolsStatus.message + || `${model.label} is not set up yet.`; + + const currentInpaintRange = sourceClip + ? getClipInpaintRange(useDAWStore.getState().timeSelection, sourceClip) + : null; + + useEffect(() => { + if (!showAIClipGeneration || !sourceClip) return; + + const nextRange = workflow.id === "inpaint-selection" ? currentInpaintRange : null; + const sameRange = + (!nextRange && !aiClipGenerationRange) + || ( + nextRange + && aiClipGenerationRange + && Math.abs(nextRange.start - aiClipGenerationRange.start) < 0.001 + && Math.abs(nextRange.end - aiClipGenerationRange.end) < 0.001 + ); + if (!sameRange) { + setAIClipGenerationRange(nextRange); + } + + const nextParams = { + ...params, + duration: sourceClip.duration, + ...(nextRange + ? { + repainting_start: nextRange.start, + repainting_end: nextRange.end, + inpaint_start: nextRange.start, + inpaint_end: nextRange.end, + } + : {}), + }; + const normalized = normalizeWorkflowParams(workflow.id, nextParams, aiClipGenerationModelId); + if (JSON.stringify(normalized) !== JSON.stringify(params)) { + setAIClipGenerationParams(normalized); + } + }, [ + aiClipGenerationModelId, + aiClipGenerationRange, + currentInpaintRange, + params, + setAIClipGenerationParams, + setAIClipGenerationRange, + showAIClipGeneration, + sourceClip, + workflow.id, + ]); + + const stopPolling = () => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + }; + + useEffect(() => stopPolling, []); + + const paramsBySection = useMemo(() => { + return SECTION_ORDER.map((section) => ({ + section, + params: workflow.params.filter((param) => ( + param.section === section && shouldShowParam(workflow.id, param) + )), + })).filter((group) => group.params.length > 0); + }, [workflow.id, workflow.params]); + + const primaryParamGroups = paramsBySection.filter((group) => !ADVANCED_SECTIONS.has(group.section)); + const advancedParamGroups = paramsBySection.filter((group) => ADVANCED_SECTIONS.has(group.section)); + const advancedControlCount = advancedParamGroups.reduce((count, group) => count + group.params.length, 0); + const inpaintSelectionMissing = workflow.id === "inpaint-selection" && !aiClipGenerationRange; + const stableSourcePromptMissing = + aiClipGenerationModelId === STABLE_AUDIO_3_MODEL_ID + && STABLE_SOURCE_WORKFLOW_IDS.has(workflow.id) + && String(params.prompt ?? "").trim().length === 0; + const progressChips = [ + progress.backend ? progress.backend.toUpperCase() : "", + progress.runMode ? progress.runMode.toUpperCase() : "", + formatSessionModeLabel(progress.sessionMode), + formatRuntimeProfileLabel(progress.runtimeProfile), + ].filter(Boolean); + const hasProgressDetails = Boolean(progress.statusNote || progressChips.length || progress.lastStderrLine || progress.lastStdoutLine); + + const handleParamChange = (key: string, value: unknown) => { + setAIClipGenerationParams( + normalizeWorkflowParams(workflow.id, { + ...params, + [key]: value, + }, aiClipGenerationModelId), + ); + }; + + const handleCancel = async () => { + if (isGenerating) { + completedRef.current = true; + stopPolling(); + await nativeBridge.cancelAIGeneration(); + setIsGenerating(false); + setProgress({ state: "idle", progress: 0 }); + return; + } + closeAIClipGeneration(); + }; + + const handleGenerate = async () => { + if (!sourceTrack || !sourceClip || !aiClipGenerationWorkflowId) { + setAIClipGenerationError("Source clip is no longer available."); + return; + } + if (!isModelReady) { + openAiToolsSetup("audioGeneration"); + return; + } + if (stableSourcePromptMissing) { + setAIClipGenerationError("Stable Audio source workflows need a direction prompt."); + return; + } + if (inpaintSelectionMissing) { + setAIClipGenerationError("Create a time selection that overlaps this clip before inpainting."); + return; + } + + completedRef.current = false; + setIsGenerating(true); + setProgress({ state: "loading", progress: 0.01, phase: "starting" }); + setAIClipGenerationError(""); + + const extensionDuration = Number(params.extension_duration ?? 20); + const requestParams = { + ...params, + source: { + filePath: sourceClip.filePath, + clipOffset: sourceClip.offset || 0, + clipDuration: sourceClip.duration, + sourceTrackId: sourceTrack.id, + sourceTrackName: sourceTrack.name, + sourceClipId: sourceClip.id, + sourceClipName: sourceClip.name, + sourceClipStartTime: sourceClip.startTime, + inpaintRange: aiClipGenerationRange, + extensionDuration, + }, + }; + + try { + const result = await nativeBridge.startAIGeneration( + sourceTrack.id, + aiClipGenerationModelId, + workflow.id, + requestParams, + ); + + if (!result.started) { + setIsGenerating(false); + setProgress({ state: "error", progress: 0, error: result.error }); + setAIClipGenerationError(result.error || "Failed to start AI generation."); + return; + } + + let idleCount = 0; + pollRef.current = setInterval(async () => { + if (completedRef.current) return; + const nextProgress = await nativeBridge.getAIGenerationProgress(); + if (completedRef.current) return; + setProgress(nextProgress); + + if (nextProgress.state === "idle") { + idleCount += 1; + if (idleCount >= 10) { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError("Generation did not start. Open AI Tools Setup and check the selected model."); + } + return; + } + idleCount = 0; + + if (nextProgress.state === "done") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + if (!nextProgress.outputFile) { + setAIClipGenerationError("Generation finished without producing an audio file."); + return; + } + await addGeneratedSourceAudioClip({ + sourceTrackId: sourceTrack.id, + sourceClipId: sourceClip.id, + workflowId: workflow.id, + filePath: nextProgress.outputFile, + }); + setTimeout(() => closeAIClipGeneration(), 500); + } else if (nextProgress.state === "error") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError(nextProgress.error || nextProgress.message || "Generation failed."); + } else if (nextProgress.state === "cancelled") { + completedRef.current = true; + stopPolling(); + setIsGenerating(false); + } + }, 250); + } catch (error) { + stopPolling(); + setIsGenerating(false); + setAIClipGenerationError(error instanceof Error ? error.message : "Generation failed."); + } + }; + + const renderParam = (param: AIWorkflowParam) => { + const value = params[param.key]; + const isRangeParam = RANGE_PARAM_KEYS.has(param.key); + + if (param.type === "textarea") { + return ( +