From 0f94fa1aa5fd1f3ab98cce3275736db6c76d1b46 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sat, 10 Oct 2026 15:34:36 -0400 Subject: [PATCH 1/5] Reduce matrix scheduling time and recover publish retries --- app-e2e/src/Test/E2E/Endpoint/Publish.purs | 11 ++- app/src/App/API.purs | 27 +++---- app/src/App/Server/JobExecutor.purs | 54 ++++---------- app/src/App/Server/MatrixBuilder.purs | 82 ++++++++++++++-------- app/test/App/API.purs | 12 +++- app/test/App/SQLite.js | 13 +++- app/test/App/SQLite.purs | 9 ++- app/test/App/Server/MatrixBuilder.purs | 74 +++++++++++++++++-- 8 files changed, 181 insertions(+), 101 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Publish.purs b/app-e2e/src/Test/E2E/Endpoint/Publish.purs index ce5e96db..b790651a 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Publish.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Publish.purs @@ -121,7 +121,7 @@ spec = do Assert.shouldEqual completedDuplicate.jobId head.jobId Assert.shouldEqual completedDuplicate.disposition (Just V1.AlreadyPublishedSubmission) - Spec.it "reports an already-published package as idempotent success" do + Spec.it "repairs matrix scheduling for an already-published package without republishing" do created <- Client.publish Fixtures.effectAlreadyPublishedData Assert.shouldEqual created.disposition (Just V1.Created) job <- Env.pollJobOrFail created.jobId @@ -131,6 +131,15 @@ spec = do Assert.shouldEqual details.error Nothing _ -> Assert.fail "Expected a publish job." + -- The fixture is fully published but this test starts with no job history. + -- An idempotent retry must still enqueue its missing compiler checks. + jobs <- Client.getJobsWith Client.IncludeCompleted + let payload = Fixtures.effectAlreadyPublishedData + unless (Array.any (Env.isMatrixJobFor { name: payload.name, version: payload.version }) jobs) do + Assert.fail "Expected an already-published retry to enqueue matrix work." + uploaded <- Env.hasStorageUpload { name: payload.name, version: payload.version } + Assert.shouldEqual uploaded false + duplicate <- Client.publish Fixtures.effectAlreadyPublishedData Assert.shouldEqual duplicate.jobId created.jobId Assert.shouldEqual duplicate.disposition (Just V1.AlreadyPublishedSubmission) diff --git a/app/src/App/API.purs b/app/src/App/API.purs index f1753d21..0f8a6ca6 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -354,12 +354,6 @@ type PublishResult = , matrix :: Maybe PublishedPackage } -toPublishResult :: Maybe PublishedPackage -> PublishResult -toPublishResult matrix = - { disposition: if isJust matrix then Published else AlreadyPublished - , matrix - } - -- | Resolve both compiler and resolutions for a publish operation. -- | Will come up with some sort of plan if not provided with a compiler and/or resolutions. resolveCompilerAndDeps @@ -747,7 +741,7 @@ publish payload = do } _, _ -> do Log.info "Verifying the package build plan..." - compilerIndex <- MatrixBuilder.readCompilerIndex + { compilerIndex } <- MatrixBuilder.readMatrixSnapshot resolveCompilerAndDeps compilerIndex (Manifest receivedManifest) payload.compiler payload.resolutions Log.info $ "Using compiler " <> Version.print compiler @@ -787,11 +781,8 @@ publish payload = do Registry.writeManifest (Manifest receivedManifest) Log.notice "Verified the existing package tarball and reconciled its registry metadata and manifest." - reconciledResult = - if isNothing existingManifest then - Just { compiler, dependencies: receivedManifest.dependencies, version: receivedManifest.version } - else - Nothing + matrix = { compiler, dependencies: receivedManifest.dependencies, version: receivedManifest.version } + reconciledDisposition = if isNothing existingManifest then Published else AlreadyPublished case existingPublishedVersion of Just _ | Just url <- existingPursuitUrl -> do @@ -800,7 +791,9 @@ publish payload = do , url ] FS.Extra.remove tmp - pure $ toPublishResult Nothing + -- Scheduling can have been interrupted after publication. An idempotent + -- retry must still return follow-up work without claiming a new publication. + pure { disposition: AlreadyPublished, matrix: Just matrix } -- If metadata already contains this version, first reconcile storage and -- both Git repositories, then retry any missing Pursuit documentation. @@ -813,7 +806,7 @@ publish payload = do , ". Please try with a later compiler." ] FS.Extra.remove tmp - pure $ toPublishResult reconciledResult + pure { disposition: reconciledDisposition, matrix: Just matrix } Just info -> do reconcileExistingPublication info @@ -849,7 +842,7 @@ publish payload = do Right _ -> do FS.Extra.remove tmp Log.notice "Successfully uploaded package docs to Pursuit! 🎉 🚀" - pure $ toPublishResult reconciledResult + pure { disposition: reconciledDisposition, matrix: Just matrix } -- In this case the package version has not been published, so we proceed -- with ordinary publishing. @@ -1056,7 +1049,7 @@ publish payload = do ] FS.Extra.remove tmp - pure $ toPublishResult $ Just { compiler, dependencies: receivedManifest.dependencies, version: receivedManifest.version } + pure { disposition: Published, matrix: Just matrix } validateResolutions :: forall r. Manifest -> Map PackageName Version -> Run (EXCEPT String + r) Unit validateResolutions manifest resolutions = do @@ -1115,7 +1108,7 @@ findAllCompilers . { source :: FilePath, manifest :: Manifest, compilers :: NonEmptyArray Version } -> Run (REGISTRY_READ + STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult findAllCompilers { source, manifest, compilers } = do - compilerIndex <- MatrixBuilder.readCompilerIndex + { compilerIndex } <- MatrixBuilder.readMatrixSnapshot checkedCompilers <- for compilers \target -> do Log.debug $ "Trying compiler " <> Version.print target case Solver.solveWithCompiler (Range.exact target) compilerIndex (un Manifest manifest).dependencies of diff --git a/app/src/App/Server/JobExecutor.purs b/app/src/App/Server/JobExecutor.purs index 346d3297..9408c19b 100644 --- a/app/src/App/Server/JobExecutor.purs +++ b/app/src/App/Server/JobExecutor.purs @@ -10,7 +10,6 @@ import Data.Array as Array import Data.DateTime (DateTime) import Data.Int as Int import Data.Map as Map -import Data.Set as Set import Data.Time.Duration as Duration import Effect.Aff as Aff import Effect.Ref as Ref @@ -26,6 +25,7 @@ import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.App.Server.Env (ServerEffects, ServerEnv, runEffects) import Registry.App.Server.Env as Env +import Registry.App.Server.MatrixBuilder (MatrixSolverResult) import Registry.App.Server.MatrixBuilder as MatrixBuilder import Registry.ManifestIndex as ManifestIndex import Registry.PackageName as PackageName @@ -157,26 +157,10 @@ executeJob _ = case _ of -- At this point this package has been verified with one compiler only. -- So we need to enqueue compilation jobs for (1) same package, all the other -- compilers, and (2) same compiler, all packages that depend on this one - -- TODO here we are building the compiler index, but we should really cache it - compilerIndex <- MatrixBuilder.readCompilerIndex - let solverData = { compiler, name, version, dependencies, compilerIndex } - samePackageAllCompilers <- MatrixBuilder.solveForAllCompilers solverData - sameCompilerAllDependants <- MatrixBuilder.solveDependantsForCompiler solverData - for (Array.fromFoldable $ Set.union samePackageAllCompilers sameCompilerAllDependants) \{ compiler: solvedCompiler, resolutions, name: solvedPackage, version: solvedVersion } -> do - Log.info $ Array.fold - [ "Enqueuing matrix job: compiler " - , Version.print solvedCompiler - , ", package " - , PackageName.print solvedPackage - , "@" - , Version.print solvedVersion - ] - Db.insertMatrixJob - { payload: resolutions - , compilerVersion: solvedCompiler - , packageName: solvedPackage - , packageVersion: solvedVersion - } + snapshot <- MatrixBuilder.readMatrixSnapshot + let solverData = { compiler, name, version, dependencies, snapshot } + void $ MatrixBuilder.solveForAllCompilers solverData enqueueMatrixJob + void $ MatrixBuilder.solveDependantsForCompiler solverData enqueueMatrixJob pure $ Just result.disposition UnpublishJob { payload } -> API.authenticated payload $> Nothing TransferJob { payload } -> API.authenticated payload $> Nothing @@ -186,28 +170,18 @@ executeJob _ = case _ of -- packages trigger jobs for their dependents so it cascades through the registry) dependencies <- MatrixBuilder.runMatrixJob details - -- TODO here we are building the compiler index, but we should really cache it - compilerIndex <- MatrixBuilder.readCompilerIndex - let solverData = { compiler: details.compilerVersion, name: packageName, version: packageVersion, dependencies, compilerIndex } - sameCompilerAllDependants <- MatrixBuilder.solveDependantsForCompiler solverData - for_ (Array.fromFoldable sameCompilerAllDependants) \{ compiler: solvedCompiler, resolutions, name: solvedPackage, version: solvedVersion } -> do - Log.info $ Array.fold - [ "Enqueuing matrix job: compiler " - , Version.print solvedCompiler - , ", package " - , PackageName.print solvedPackage - , "@" - , Version.print solvedVersion - ] - Db.insertMatrixJob - { payload: resolutions - , compilerVersion: solvedCompiler - , packageName: solvedPackage - , packageVersion: solvedVersion - } + snapshot <- MatrixBuilder.readMatrixSnapshot + let solverData = { compiler: details.compilerVersion, name: packageName, version: packageVersion, dependencies, snapshot } + void $ MatrixBuilder.solveDependantsForCompiler solverData enqueueMatrixJob pure Nothing PackageSetJob payload -> API.packageSetUpdate payload $> Nothing +enqueueMatrixJob :: forall r. MatrixSolverResult -> Run (DB + LOG + r) Unit +enqueueMatrixJob { compiler, resolutions, name, version } = do + Log.info $ "Enqueuing matrix job: compiler " <> Version.print compiler <> ", package " <> formatPackageVersion name version + void $ Db.insertMatrixJob + { payload: resolutions, compilerVersion: compiler, packageName: name, packageVersion: version } + upgradeRegistryToNewCompiler :: forall r. Version -> Run (DB + LOG + EXCEPT String + REGISTRY_READ + r) Unit upgradeRegistryToNewCompiler newCompilerVersion = do Log.info $ "New compiler found: " <> Version.print newCompilerVersion diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index 6e1ebc00..10b31962 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -1,9 +1,11 @@ module Registry.App.Server.MatrixBuilder ( BuildPlanEntry + , MatrixSnapshot + , MatrixSolverResult , checkIfNewCompiler , installBuildPlan , printCompilerFailure - , readCompilerIndex + , readMatrixSnapshot , resolutionsToBuildPlan , runMatrixJob , solveForAllCompilers @@ -36,6 +38,7 @@ import Registry.App.Effect.Storage (STORAGE) import Registry.App.Effect.Storage as Storage import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.Tmp as Tmp +import Registry.ManifestIndex (ManifestIndex) import Registry.ManifestIndex as ManifestIndex import Registry.Metadata as Metadata import Registry.PackageName as PackageName @@ -100,14 +103,21 @@ runMatrixJob { compilerVersion, packageName, packageVersion, payload: buildPlan Log.error $ "No existing metadata for " <> PackageName.print packageName <> "@" <> Version.print packageVersion Except.throw $ "No manifest found for " <> PackageName.print packageName <> "@" <> Version.print packageVersion --- TODO feels like we should be doing this at startup and use the cache instead --- of reading files all over again -readCompilerIndex :: forall r. Run (REGISTRY_READ + AFF + EXCEPT String + r) Solver.CompilerIndex -readCompilerIndex = do +-- | A pass-local snapshot, captured after publication or compatibility writes. +-- | The executor is serial; queued jobs cannot change it during scheduling. +type MatrixSnapshot = + { compilerIndex :: Solver.CompilerIndex + , manifestIndex :: ManifestIndex + , metadata :: Map PackageName Metadata + , compilers :: NonEmptyArray Version + } + +readMatrixSnapshot :: forall r. Run (REGISTRY_READ + AFF + EXCEPT String + r) MatrixSnapshot +readMatrixSnapshot = do metadata <- Registry.readAllMetadata - manifests <- Registry.readAllManifests - allCompilers <- PursVersions.pursVersions - pure $ Solver.buildCompilerIndex allCompilers manifests metadata + manifestIndex <- Registry.readAllManifests + compilers <- PursVersions.pursVersions + pure { compilerIndex: Solver.buildCompilerIndex compilers manifestIndex metadata, manifestIndex, metadata, compilers } -- | A build plan entry with integrity information for verification. type BuildPlanEntry = { version :: Version, hash :: Sha256, bytes :: Number } @@ -168,7 +178,7 @@ printCompilerFailure compiler = case _ of ] type MatrixSolverData = - { compilerIndex :: Solver.CompilerIndex + { snapshot :: MatrixSnapshot , compiler :: Version , name :: PackageName , version :: Version @@ -182,19 +192,24 @@ type MatrixSolverResult = , resolutions :: Map PackageName Version } -solveForAllCompilers :: forall r. MatrixSolverData -> Run (AFF + EXCEPT String + LOG + r) (Set MatrixSolverResult) -solveForAllCompilers solverData@{ compiler } = do +-- | Emit each solved plan before continuing, so cancellation does not discard +-- | all progress. Callers can persist plans using the existing matrix queue. +solveForAllCompilers :: forall r. MatrixSolverData -> (MatrixSolverResult -> Run (LOG + r) Unit) -> Run (LOG + r) (Set MatrixSolverResult) +solveForAllCompilers solverData@{ compiler, snapshot } emit = do -- remove the compiler we tested with from the set of all of them - compilers <- (Array.filter (_ /= compiler) <<< NonEmptyArray.toArray) <$> PursVersions.pursVersions - newJobs <- for compilers \target -> - trySolveForCompiler (solverData { compiler = target }) + let compilers = Array.filter (_ /= compiler) $ NonEmptyArray.toArray snapshot.compilers + newJobs <- for compilers \target -> do + result <- trySolveForCompiler (solverData { compiler = target }) + for_ result emit + pure result pure $ Set.fromFoldable $ Array.catMaybes newJobs -solveDependantsForCompiler :: forall r. MatrixSolverData -> Run (EXCEPT String + LOG + REGISTRY_READ + r) (Set MatrixSolverResult) -solveDependantsForCompiler { compilerIndex, name, version, compiler } = do - manifestIndex <- Registry.readAllManifests +solveDependantsForCompiler :: forall r. MatrixSolverData -> (MatrixSolverResult -> Run (LOG + r) Unit) -> Run (LOG + r) (Set MatrixSolverResult) +solveDependantsForCompiler { snapshot, name, version, compiler } emit = do let seed = Tuple name version - { results, visited } <- go manifestIndex (Set.singleton seed) name version + -- Sort this fixed index once, not once per already-compatible dependant. + let manifests = ManifestIndex.toSortedArray ManifestIndex.ConsiderRanges snapshot.manifestIndex + { results, visited } <- go manifests (Set.singleton seed) name version Log.info $ Array.fold [ "Cascade from " , PackageName.print name @@ -202,7 +217,7 @@ solveDependantsForCompiler { compilerIndex, name, version, compiler } = do , Version.print version , ": " , show (Set.size results) - , " enqueued out of " + , " solved out of " , show (Set.size visited - 1) , " dependants visited" ] @@ -219,17 +234,20 @@ solveDependantsForCompiler { compilerIndex, name, version, compiler } = do -- never be retriggered. -- With this recursive propagation, when C@new completes we cascade through -- B (already compiled) and reach A, allowing for a plan to resolve. - go manifestIndex visited pkgName pkgVersion = do - let dependentManifests = ManifestIndex.dependants manifestIndex pkgName pkgVersion - foldM (processManifest manifestIndex) { visited, results: Set.empty } dependentManifests + go manifests visited pkgName pkgVersion = do + let + dependentManifests = Array.filter + (\(Manifest manifest) -> maybe false (flip Range.includes pkgVersion) $ Map.lookup pkgName manifest.dependencies) + manifests + foldM (processManifest manifests) { visited, results: Set.empty } dependentManifests - processManifest manifestIndex acc (Manifest manifest) = do + processManifest manifests acc (Manifest manifest) = do let pv = Tuple manifest.name manifest.version if Set.member pv acc.visited then pure acc else do let newVisited = Set.insert pv acc.visited - Registry.readMetadata manifest.name >>= case _ of + case Map.lookup manifest.name snapshot.metadata of Nothing -> do Log.warn $ "No metadata for dependant " <> PackageName.print manifest.name <> ", skipping" pure { visited: newVisited, results: acc.results } @@ -241,21 +259,23 @@ solveDependantsForCompiler { compilerIndex, name, version, compiler } = do Just { compilers } | elem compiler compilers -> do -- Already has compiler: propagate through to find stranded packages - sub <- go manifestIndex newVisited manifest.name manifest.version + sub <- go manifests newVisited manifest.name manifest.version pure { visited: sub.visited, results: acc.results <> sub.results } | otherwise -> do - result <- trySolveForCompiler { compilerIndex, compiler, name: manifest.name, version: manifest.version, dependencies: manifest.dependencies } - pure case result of - Nothing -> { visited: newVisited, results: acc.results } - Just entry -> { visited: newVisited, results: Set.insert entry acc.results } + result <- trySolveForCompiler { snapshot, compiler, name: manifest.name, version: manifest.version, dependencies: manifest.dependencies } + case result of + Nothing -> pure { visited: newVisited, results: acc.results } + Just entry -> do + emit entry + pure { visited: newVisited, results: Set.insert entry acc.results } -- | Try to solve a package's dependencies for a specific compiler. Returns -- | the solver result if the produced build plan targets the expected compiler, -- | Nothing otherwise (solver failure or compiler mismatch). trySolveForCompiler :: forall r. MatrixSolverData -> Run (LOG + r) (Maybe MatrixSolverResult) -trySolveForCompiler { compilerIndex, compiler, name, version, dependencies } = do +trySolveForCompiler { snapshot, compiler, name, version, dependencies } = do Log.debug $ "Trying compiler " <> Version.print compiler <> " for package " <> PackageName.print name - case Solver.solveWithCompiler (Range.exact compiler) compilerIndex dependencies of + case Solver.solveWithCompiler (Range.exact compiler) snapshot.compilerIndex dependencies of Left solverErrors -> do Log.info $ "Failed to solve with compiler " <> Version.print compiler <> ": " <> PackageName.print name <> "@" <> Version.print version Log.debug $ "Solver errors:\n" <> foldMapWithIndex diff --git a/app/test/App/API.purs b/app/test/App/API.purs index a165504c..1f57e5c3 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -177,7 +177,7 @@ spec = do } -- First, we publish the package. - void $ API.publish publishArgs + publication <- API.publish publishArgs -- Then, we can check that it did make it to "Pursuit" as expected Pursuit.getPublishedVersions name >>= case _ of @@ -205,9 +205,15 @@ spec = do unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion name version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') - -- Finally, publishing the same package again is an idempotent success. + -- Retrying after publication but before matrix scheduling must recover + -- the same follow-up work while remaining an idempotent success. Except.runExcept (API.publish publishArgs) >>= case _ of - Right { disposition: V1.AlreadyPublished } -> pure unit + Right { disposition: V1.AlreadyPublished, matrix } -> do + unless (isJust matrix && matrix == publication.matrix) do + Except.throw "Expected an already-published retry to recover the original matrix follow-up." + Registry.readMetadata name >>= \after -> + unless (after == Just (Metadata effectMetadata)) do + Except.throw "Retrying publication changed the recorded package metadata." Right _ -> Except.throw $ "Expected publishing " <> formatPackageVersion name version <> " twice to report an already-published disposition." Left err -> Except.throw $ "Expected an idempotent publish but got error: " <> err diff --git a/app/test/App/SQLite.js b/app/test/App/SQLite.js index 6d547f54..7469058f 100644 --- a/app/test/App/SQLite.js +++ b/app/test/App/SQLite.js @@ -1,11 +1,18 @@ import Database from "better-sqlite3"; import { readFileSync } from "node:fs"; -export const withDatabase = (test) => () => { +export const openDatabase = () => { const db = new Database(":memory:"); + db.pragma("foreign_keys = ON"); + db.exec(readFileSync("db/schema.sql", "utf8")); + return db; +}; + +export const closeDatabase = db => () => db.close(); + +export const withDatabase = (test) => () => { + const db = openDatabase(); try { - db.pragma("foreign_keys = ON"); - db.exec(readFileSync("db/schema.sql", "utf8")); test(db)(); } finally { db.close(); diff --git a/app/test/App/SQLite.purs b/app/test/App/SQLite.purs index f313ce9a..857ac2b4 100644 --- a/app/test/App/SQLite.purs +++ b/app/test/App/SQLite.purs @@ -1,7 +1,8 @@ -module Test.Registry.App.SQLite (spec) where +module Test.Registry.App.SQLite (spec, withDatabaseAff) where import Registry.App.Prelude +import Effect.Aff as Aff import Effect.Exception as Exception import Registry.API.V1 (SortOrder(..)) import Registry.API.V1 as V1 @@ -14,6 +15,12 @@ import Test.Spec as Spec foreign import withDatabase :: (SQLite -> Effect Unit) -> Effect Unit +foreign import openDatabase :: Effect SQLite +foreign import closeDatabase :: SQLite -> Effect Unit + +withDatabaseAff :: forall a. (SQLite -> Aff a) -> Aff a +withDatabaseAff = Aff.bracket (liftEffect openDatabase) (liftEffect <<< closeDatabase) + spec :: Spec.Spec Unit spec = Spec.it "allows failed publishes to be retried without losing deduplication or job history" $ liftEffect $ withDatabase \db -> do let diff --git a/app/test/App/Server/MatrixBuilder.purs b/app/test/App/Server/MatrixBuilder.purs index f8986502..518d5588 100644 --- a/app/test/App/Server/MatrixBuilder.purs +++ b/app/test/App/Server/MatrixBuilder.purs @@ -2,16 +2,24 @@ module Test.Registry.App.Server.MatrixBuilder (spec) where import Registry.App.Prelude +import Data.Array as Array import Data.Array.NonEmpty as NonEmptyArray import Data.Map as Map import Data.Set as Set +import Effect.Aff as Aff import Effect.Ref as Ref +import Registry.API.V1 (Job(..), SortOrder(..)) +import Registry.API.V1 as V1 +import Registry.App.SQLite as SQLite import Registry.App.Server.MatrixBuilder as MatrixBuilder import Registry.ManifestIndex as ManifestIndex import Registry.PackageName as PackageName import Registry.Solver as Solver -import Registry.Test.Assert.Run (runRegistryMock) +import Registry.Test.Assert.Run (runBaseEffects, runRegistryMock) import Registry.Test.Utils as Utils +import Run as Run +import Run.Except as Except +import Test.Registry.App.SQLite (withDatabaseAff) import Test.Spec as Spec import Test.Spec.Assertions as Assert @@ -77,9 +85,10 @@ spec = do -- Rebuild CompilerIndex from current metadata (as JobExecutor does) currentMetadata <- liftEffect $ Ref.read metadataRef let compilerIndex = Solver.buildCompilerIndex allCompilers cascadeIndex currentMetadata - let solverData = { compilerIndex, compiler: compiler_0_15_11, name, version, dependencies } + let snapshot = { compilerIndex, manifestIndex: cascadeIndex, metadata: currentMetadata, compilers: allCompilers } + let solverData = { snapshot, compiler: compiler_0_15_11, name, version, dependencies } runRegistryMock metadataRef indexRef - $ MatrixBuilder.solveDependantsForCompiler solverData + $ MatrixBuilder.solveDependantsForCompiler solverData (const $ pure unit) -- Wave 0: leaves complete -- prelude@5.0.0 -> should find leaf-dep@4.0.0 (prelude >=4.0.0 <6.0.0 includes 5.0.0) @@ -115,6 +124,60 @@ spec = do Assert.fail $ "Step 5: Expected top-pkg via propagation through mid-pkg, got: " <> show (Set.map PackageName.print r5Names) + Spec.it "preserves queued plans on interruption and resumes without duplicates" do + let + initial = setup [ "0.15.10" ] + index = Utils.fromRight "Invalid test index" $ ManifestIndex.insert ManifestIndex.ConsiderRanges leafDep6Manifest initial.index + metadata = Map.union initial.metadata $ Map.fromFoldable + [ Utils.unsafeMetadata "leaf-dep" [ Tuple "6.0.0" [ "0.15.10" ] ] ] + compiler12 = Utils.unsafeVersion "0.15.12" + compilers = Utils.unsafeNonEmptyArray [ compiler_0_15_10, compiler_0_15_11, compiler12 ] + snapshot = { compilerIndex: Solver.buildCompilerIndex compilers index metadata, manifestIndex: index, metadata, compilers } + solverData = initial.solverData { snapshot = snapshot } + expected name version compiler resolutions = { name, version, compiler, resolutions } + cases = + [ { solve: MatrixBuilder.solveForAllCompilers + , plans: + [ expected preludeName preludeVersion compiler_0_15_10 Map.empty + , expected preludeName preludeVersion compiler12 Map.empty + ] + } + , { solve: MatrixBuilder.solveDependantsForCompiler + , plans: + [ expected effectName (Utils.unsafeVersion "4.0.0") compiler_0_15_11 (Map.singleton preludeName preludeVersion) + , expected leafDepName leafDep6Version compiler_0_15_11 (Map.singleton preludeName preludeVersion) + ] + } + ] + for_ cases \{ solve, plans } -> liftAff $ withDatabaseAff \db -> do + let + enqueue { name, version, compiler, resolutions } = Run.liftEffect $ void $ SQLite.insertMatrixJob db + { packageName: name, packageVersion: version, compilerVersion: compiler, payload: resolutions } + queued = liftEffect $ SQLite.selectJobs db { since: bottom, until: top, order: ASC, includeCompleted: false } + interrupted <- Aff.attempt $ runBaseEffects $ solve solverData \plan -> do + enqueue plan + Except.throw "interrupted" + case interrupted of + Left error | Aff.message error == "interrupted" -> pure unit + _ -> Assert.fail "Expected scheduling to be interrupted after its first enqueue." + partial <- queued + Assert.shouldEqual (Array.length partial.jobs) 1 + -- A retry must retain the original job, enqueue the remaining plan, + -- and preserve the compiler and complete dependency resolutions. + void $ runBaseEffects $ solve solverData enqueue + complete <- queued + let + actual = Array.mapMaybe + ( case _ of + MatrixJob j -> Just $ expected j.packageName j.packageVersion j.compilerVersion j.payload + _ -> Nothing + ) + complete.jobs + unless (Array.null complete.failed && Array.length actual == 2 && Set.fromFoldable actual == Set.fromFoldable plans) do + Assert.fail "Expected exactly the two solved build plans, without duplicates or altered resolutions." + unless (all (\old -> Array.any (\new -> (V1.jobInfo new).jobId == (V1.jobInfo old).jobId) complete.jobs) partial.jobs) do + Assert.fail "Retry replaced a previously queued job." + where preludeName = Utils.unsafePackageName "prelude" effectName = Utils.unsafePackageName "effect" @@ -152,9 +215,10 @@ spec = do ] compilerIndex = Solver.buildCompilerIndex allCompilers index metadata + snapshot = { compilerIndex, manifestIndex: index, metadata, compilers: allCompilers } solverData = - { compilerIndex + { snapshot , compiler: compiler_0_15_11 , name: preludeName , version: preludeVersion @@ -207,4 +271,4 @@ spec = do indexRef <- liftEffect $ Ref.new index metadataRef <- liftEffect $ Ref.new metadata runRegistryMock metadataRef indexRef - $ MatrixBuilder.solveDependantsForCompiler solverData + $ MatrixBuilder.solveDependantsForCompiler solverData (const $ pure unit) From 6a60348d5c49970b52a58e48fe49d9c9f450bc2b Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sat, 10 Oct 2026 20:22:21 -0400 Subject: [PATCH 2/5] Remove publication retry matrix recovery --- app-e2e/src/Test/E2E/Endpoint/Publish.purs | 11 +---------- app/src/App/API.purs | 14 ++++++++------ app/test/App/API.purs | 14 ++++---------- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Publish.purs b/app-e2e/src/Test/E2E/Endpoint/Publish.purs index b790651a..ce5e96db 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Publish.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Publish.purs @@ -121,7 +121,7 @@ spec = do Assert.shouldEqual completedDuplicate.jobId head.jobId Assert.shouldEqual completedDuplicate.disposition (Just V1.AlreadyPublishedSubmission) - Spec.it "repairs matrix scheduling for an already-published package without republishing" do + Spec.it "reports an already-published package as idempotent success" do created <- Client.publish Fixtures.effectAlreadyPublishedData Assert.shouldEqual created.disposition (Just V1.Created) job <- Env.pollJobOrFail created.jobId @@ -131,15 +131,6 @@ spec = do Assert.shouldEqual details.error Nothing _ -> Assert.fail "Expected a publish job." - -- The fixture is fully published but this test starts with no job history. - -- An idempotent retry must still enqueue its missing compiler checks. - jobs <- Client.getJobsWith Client.IncludeCompleted - let payload = Fixtures.effectAlreadyPublishedData - unless (Array.any (Env.isMatrixJobFor { name: payload.name, version: payload.version }) jobs) do - Assert.fail "Expected an already-published retry to enqueue matrix work." - uploaded <- Env.hasStorageUpload { name: payload.name, version: payload.version } - Assert.shouldEqual uploaded false - duplicate <- Client.publish Fixtures.effectAlreadyPublishedData Assert.shouldEqual duplicate.jobId created.jobId Assert.shouldEqual duplicate.disposition (Just V1.AlreadyPublishedSubmission) diff --git a/app/src/App/API.purs b/app/src/App/API.purs index 0f8a6ca6..5727a0ac 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -782,7 +782,11 @@ publish payload = do Log.notice "Verified the existing package tarball and reconciled its registry metadata and manifest." matrix = { compiler, dependencies: receivedManifest.dependencies, version: receivedManifest.version } - reconciledDisposition = if isNothing existingManifest then Published else AlreadyPublished + reconciledResult = + if isNothing existingManifest then + { disposition: Published, matrix: Just matrix } + else + { disposition: AlreadyPublished, matrix: Nothing } case existingPublishedVersion of Just _ | Just url <- existingPursuitUrl -> do @@ -791,9 +795,7 @@ publish payload = do , url ] FS.Extra.remove tmp - -- Scheduling can have been interrupted after publication. An idempotent - -- retry must still return follow-up work without claiming a new publication. - pure { disposition: AlreadyPublished, matrix: Just matrix } + pure { disposition: AlreadyPublished, matrix: Nothing } -- If metadata already contains this version, first reconcile storage and -- both Git repositories, then retry any missing Pursuit documentation. @@ -806,7 +808,7 @@ publish payload = do , ". Please try with a later compiler." ] FS.Extra.remove tmp - pure { disposition: reconciledDisposition, matrix: Just matrix } + pure reconciledResult Just info -> do reconcileExistingPublication info @@ -842,7 +844,7 @@ publish payload = do Right _ -> do FS.Extra.remove tmp Log.notice "Successfully uploaded package docs to Pursuit! 🎉 🚀" - pure { disposition: reconciledDisposition, matrix: Just matrix } + pure reconciledResult -- In this case the package version has not been published, so we proceed -- with ordinary publishing. diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 1f57e5c3..8679e004 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -177,7 +177,7 @@ spec = do } -- First, we publish the package. - publication <- API.publish publishArgs + void $ API.publish publishArgs -- Then, we can check that it did make it to "Pursuit" as expected Pursuit.getPublishedVersions name >>= case _ of @@ -205,16 +205,10 @@ spec = do unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion name version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') - -- Retrying after publication but before matrix scheduling must recover - -- the same follow-up work while remaining an idempotent success. + -- Finally, publishing the same package again is an idempotent success. Except.runExcept (API.publish publishArgs) >>= case _ of - Right { disposition: V1.AlreadyPublished, matrix } -> do - unless (isJust matrix && matrix == publication.matrix) do - Except.throw "Expected an already-published retry to recover the original matrix follow-up." - Registry.readMetadata name >>= \after -> - unless (after == Just (Metadata effectMetadata)) do - Except.throw "Retrying publication changed the recorded package metadata." - Right _ -> Except.throw $ "Expected publishing " <> formatPackageVersion name version <> " twice to report an already-published disposition." + Right { disposition: V1.AlreadyPublished, matrix: Nothing } -> pure unit + Right _ -> Except.throw $ "Expected publishing " <> formatPackageVersion name version <> " twice to report an already-published disposition without matrix follow-up." Left err -> Except.throw $ "Expected an idempotent publish but got error: " <> err case result of From 5576c09a4c5be2a688139b925ff2b5c857d16a7e Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Mon, 12 Oct 2026 19:05:20 -0400 Subject: [PATCH 3/5] Update app/src/App/Server/MatrixBuilder.purs Co-authored-by: Fabrizio Ferrai --- app/src/App/Server/MatrixBuilder.purs | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index 10b31962..c4762881 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -207,7 +207,6 @@ solveForAllCompilers solverData@{ compiler, snapshot } emit = do solveDependantsForCompiler :: forall r. MatrixSolverData -> (MatrixSolverResult -> Run (LOG + r) Unit) -> Run (LOG + r) (Set MatrixSolverResult) solveDependantsForCompiler { snapshot, name, version, compiler } emit = do let seed = Tuple name version - -- Sort this fixed index once, not once per already-compatible dependant. let manifests = ManifestIndex.toSortedArray ManifestIndex.ConsiderRanges snapshot.manifestIndex { results, visited } <- go manifests (Set.singleton seed) name version Log.info $ Array.fold From eafb74b8516500da1845e5fe70117a40d0716595 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Mon, 12 Oct 2026 19:05:28 -0400 Subject: [PATCH 4/5] Update app/src/App/Server/MatrixBuilder.purs Co-authored-by: Fabrizio Ferrai --- app/src/App/Server/MatrixBuilder.purs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index c4762881..3b76647f 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -192,8 +192,7 @@ type MatrixSolverResult = , resolutions :: Map PackageName Version } --- | Emit each solved plan before continuing, so cancellation does not discard --- | all progress. Callers can persist plans using the existing matrix queue. +-- | Takes a callback to persist each plan as it's solved solveForAllCompilers :: forall r. MatrixSolverData -> (MatrixSolverResult -> Run (LOG + r) Unit) -> Run (LOG + r) (Set MatrixSolverResult) solveForAllCompilers solverData@{ compiler, snapshot } emit = do -- remove the compiler we tested with from the set of all of them From fab49f149250a0413d12991ebd57fb2fc780e798 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Tue, 13 Oct 2026 00:50:27 +0000 Subject: [PATCH 5/5] Clarify matrix snapshot documentation and preserve caching TODOs --- app/src/App/Server/MatrixBuilder.purs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index 3b76647f..89a0d3eb 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -103,8 +103,8 @@ runMatrixJob { compilerVersion, packageName, packageVersion, payload: buildPlan Log.error $ "No existing metadata for " <> PackageName.print packageName <> "@" <> Version.print packageVersion Except.throw $ "No manifest found for " <> PackageName.print packageName <> "@" <> Version.print packageVersion --- | A pass-local snapshot, captured after publication or compatibility writes. --- | The executor is serial; queued jobs cannot change it during scheduling. +-- | Package and compiler data used to decide which compilation jobs to queue. +-- | Read once per scheduling pass, after the package metadata has been updated. type MatrixSnapshot = { compilerIndex :: Solver.CompilerIndex , manifestIndex :: ManifestIndex @@ -112,11 +112,14 @@ type MatrixSnapshot = , compilers :: NonEmptyArray Version } +-- TODO feels like we should be doing this at startup and use the cache instead +-- of reading files all over again readMatrixSnapshot :: forall r. Run (REGISTRY_READ + AFF + EXCEPT String + r) MatrixSnapshot readMatrixSnapshot = do metadata <- Registry.readAllMetadata manifestIndex <- Registry.readAllManifests compilers <- PursVersions.pursVersions + -- TODO here we are building the compiler index, but we should really cache it pure { compilerIndex: Solver.buildCompilerIndex compilers manifestIndex metadata, manifestIndex, metadata, compilers } -- | A build plan entry with integrity information for verification.