FEAT: List the saved registry entries that couldn't be restored (part 5)

.pyrit_conf_example

@@ -161,6 +161,21 @@ allow_custom_initializers: false
 # as scenario_presets_source. Defaults to ~/.pyrit/scenario_suites.
 # scenario_suites_source: https://account.blob.core.windows.net/container/scenario_suites
 
+# Optional storage for saved targets, converters, scorers, and techniques, so they
+# come back when the backend restarts. Each kind has its own folder under the source.
+# This supports the same local directory and Azure Blob sources as
+# scenario_presets_source. If any saved entry can't be restored, the backend stays
+# not ready and lists the failures in its runtime status. Fix or delete the listed
+# files, then apply the configuration or restart the backend. Defaults to
+# ~/.pyrit/registry_recipes.
+#
+# ⚠️  WARNING: The backend rebuilds every saved entry in this source as an
+# administrator, at startup and live reinitialization. Anyone who can write to this
+# source can make the backend run a model's repository code or use its identity.
+# Restrict write access to trusted administrators, and use a dedicated folder or
+# blob prefix.
+# registry_recipes_source: https://account.blob.core.windows.net/container/registry_recipes
+
 # Upload Folder for API-Created Targets
 # -------------------------------------
 # Folder that targets created through the REST API may upload local files from

doc/getting_started/pyrit_conf.md

@@ -194,6 +194,20 @@ Suites store ordered references to scenario presets and resolve each preset's cu
 
 A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/scenario_suites`. Local writes use the same hidden lock-file and atomic replacement behavior as scenario presets. Startup scans the configured source so malformed suite documents are reported immediately without preventing the backend from starting.
 
+### `registry_recipes_source`
+
+Stores saved targets, converters, scorers, and techniques in a local directory or Azure Blob container, so they come back when the backend restarts. An Azure URI may include a blob-name prefix:
+
+```yaml
+registry_recipes_source: https://account.blob.core.windows.net/pyrit-storage/registry-recipes
+```
+
+Each kind has its own folder under the source (`targets/`, `converters/`, `scorers/`, `techniques/`), with one JSON file per saved entry named after the entry. A file holds the create request the entry was made with, and restore builds the entry through the API's create for its kind; restore refuses a file that stores a value for a parameter its class marks as a credential, such as `api_key`. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/registry_recipes`.
+
+A saved file doesn't record who saved it, so the backend rebuilds every entry in this source as an administrator, at startup and after a live apply. A file here can, for example, make the backend run a model's repository code, or use the server's identity on a storage container that the file names. Write access to this source is administrator authority on the backend: grant it only to principals you already trust as backend administrators, and use a dedicated directory or blob prefix that less-trusted users and services cannot write to.
+
+The backend restores saved entries at startup and after a live apply, once the configured initializers have registered theirs. If any saved entry can't be restored, the backend doesn't become ready (state `failed`, outcome `restore-failed`), so a scan can't quietly run without an entry it expects. Until then, only the configuration and status routes answer; the rest return 503. Administrators see each failure with its reason in `GET /api/config/runtime` and on the GUI's Configuration page; everyone else sees that an administrator has to fix saved entries. Fix or delete the listed files, or, when a folder couldn't be listed, check that the backend can read this source. Then apply the configuration, if `enable_live_reinitialization` is on and the backend runs as one worker and one replica, which live apply requires; otherwise, restart the backend. Files whose names aren't valid entry names (up to 64 lowercase letters, digits, and underscores, starting with a letter) aren't failures and don't block startup; administrators can see them under `restore_ignored` in `GET /api/config/runtime`.
+
 ### `initialization_scripts`
 
 Local paths to custom Python scripts containing `PyRITInitializer` subclasses. Paths can be absolute or relative to the current working directory.

frontend/e2e/fixtures/manual_send_backend.py

@@ -4,6 +4,7 @@
 """Offline backend for manual-send browser tests, with no real configuration or providers."""
 
 import os
+import tempfile
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from unittest.mock import AsyncMock, patch
@@ -27,19 +28,22 @@ async def _initialize_offline_async(self: ConfigurationLoader, *, raise_on_initi
 
 @asynccontextmanager
 async def _offline_lifespan_async(application: FastAPI) -> AsyncGenerator[None, None]:
-    config = ConfigurationLoader(memory_db_type="in_memory", env_files=[], env_akv_ref=[])
-    with (
-        patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True),
-        patch.object(
-            ConfigurationFileService,
-            "_read_source_async",
-            new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"),
-        ),
-        patch.object(ConfigurationLoader, "load_with_overrides", return_value=config),
-        patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async),
-    ):
-        async with lifespan(application):
-            yield
+    with tempfile.TemporaryDirectory() as recipes:
+        config = ConfigurationLoader(
+            memory_db_type="in_memory", env_files=[], env_akv_ref=[], registry_recipes_source=recipes
+        )
+        with (
+            patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True),
+            patch.object(
+                ConfigurationFileService,
+                "_read_source_async",
+                new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"),
+            ),
+            patch.object(ConfigurationLoader, "load_with_overrides", return_value=config),
+            patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async),
+        ):
+            async with lifespan(application):
+                yield
 
 
 app.router.lifespan_context = _offline_lifespan_async

frontend/src/App.test.tsx

@@ -4,6 +4,7 @@
  */
 
 import { Suspense, useEffect, useLayoutEffect, useRef, useState } from "react";
+import { fluentProviderClassNames } from "@fluentui/react-components";
 import { act, render, screen, fireEvent, waitFor } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { MemoryRouter, parsePath, useLocation, useNavigate } from "react-router";
@@ -139,6 +140,7 @@ jest.mock("./components/Layout/MainLayout", () => {
     onLabelsChange,
     operatorReadOnly,
     featureNavigation,
+    banner,
   }: {
     children: React.ReactNode;
     currentView: string;
@@ -147,10 +149,12 @@ jest.mock("./components/Layout/MainLayout", () => {
     onLabelsChange: (labels: Record<string, string>) => void;
     operatorReadOnly: boolean;
     featureNavigation?: React.ReactNode;
+    banner?: React.ReactNode;
   }) => {
     const navigate = useNavigate();
     return (
       <div data-testid="main-layout" data-current-view={currentView}>
+        {banner}
         <span data-testid="global-labels-json">{JSON.stringify(labels)}</span>
         <span data-testid="home-labels-json">{JSON.stringify(labels)}</span>
         <span data-testid="operator-read-only">{String(operatorReadOnly)}</span>
@@ -870,6 +874,18 @@ describe("App", () => {
     expect(screen.getByTestId("home-view")).toBeInTheDocument();
   });
 
+  it("says inside the app's layout and theme that saved entries couldn't be restored", async () => {
+    const { runtimeApi } = jest.requireMock("./services/api");
+    runtimeApi.getReadiness.mockResolvedValueOnce({
+      ready: false, state: "failed", generation: "first", restore_failures: 1,
+    });
+    renderApp();
+
+    const message = await screen.findByText(/Some saved entries couldn't be restored/);
+    expect(screen.getByTestId("main-layout")).toContainElement(message);
+    expect(message.closest(`.${fluentProviderClassNames.root}`)).not.toBeNull();
+  });
+
   it("renders the view named by the initial URL", () => {
     renderApp("/registry/targets");
 

frontend/src/App.tsx

@@ -869,6 +869,7 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) {
           <Joyride {...tourProps} />
           <ConnectionBannerContainer />
           <MainLayout
+            banner={<RuntimeBanner />}
             currentView={currentView}
             onNavigate={handleNavigate}
             onOpenFeedback={() => setFeedbackOpen(true)}
@@ -1106,7 +1107,6 @@ function App() {
   const operatorAlias = account?.username ? (account.username.split('@')[0]?.toLowerCase() ?? null) : null
   return (
     <RuntimeProvider>
-      <RuntimeBanner />
       <UserPreferencesProvider key={accountKey ?? 'account-loading'} accountKey={accountKey}>
         <ThemeProvider>
           <ChatDraftProvider>

frontend/src/components/Configuration/Reinitialize.test.tsx

@@ -124,6 +124,86 @@ describe('Reinitialize', () => {
     expect(screen.getByRole('button', { name: 'Reinitialize PyRIT' })).toBeDisabled()
   })
 
+  it('should let a failed restore be retried by reinitializing', async () => {
+    const user = userEvent.setup()
+    api.getRuntimeStatus.mockResolvedValue({
+      ...ready,
+      state: 'failed',
+      outcome: 'restore-failed',
+      message: "1 saved registry entry couldn't be restored.",
+    })
+    render(
+      <TestWrapper>
+        <Reinitialize version="saved-v1" hasUnsavedChanges={false} liveReinitializationEnabled />
+      </TestWrapper>,
+    )
+    expect(await screen.findByText(/Runtime: failed\. 1 saved registry entry/)).toBeInTheDocument()
+    await waitFor(() => expect(screen.getByRole('button', { name: 'Reinitialize PyRIT' })).toBeEnabled())
+    await user.click(screen.getByRole('button', { name: 'Reinitialize PyRIT' }))
+    await user.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Reinitialize PyRIT' }))
+    await waitFor(() => expect(api.reinitialize).toHaveBeenCalledWith('saved-v1'))
+  })
+
+  it('should list the saved entries that could not be restored', async () => {
+    api.getRuntimeStatus.mockResolvedValue({
+      ...ready,
+      state: 'failed',
+      outcome: 'restore-failed',
+      message: "3 saved registry entries couldn't be restored.",
+      restore_failures: [
+        {
+          kind: 'targets',
+          name: 'pool',
+          reason: 'missing_dependency',
+          message: "Uses target 'gone', which doesn't exist.",
+          version: 'v1',
+          dependency: { kind: 'targets', name: 'gone' },
+        },
+        {
+          kind: 'scorers',
+          name: 'refusal',
+          reason: 'failed_dependency',
+          message: "It uses an entry that couldn't be restored.",
+          version: 'v2',
+          dependency: { kind: 'targets', name: 'pool' },
+        },
+        {
+          kind: 'converters',
+          name: null,
+          reason: 'unreadable_storage',
+          message: "The saved converters couldn't be listed: permission denied",
+          version: null,
+          dependency: null,
+        },
+      ],
+    })
+    render(
+      <TestWrapper>
+        <Reinitialize version="saved-v1" hasUnsavedChanges={false} liveReinitializationEnabled />
+      </TestWrapper>,
+    )
+
+    const region = await screen.findByRole('region', { name: "Saved entries that couldn't be restored" })
+    expect(within(region).getByText("Target pool: Uses target 'gone', which doesn't exist.")).toBeInTheDocument()
+    expect(within(region).getByText("Scorer refusal: It uses an entry that couldn't be restored. (target pool)"))
+      .toBeInTheDocument()
+    expect(within(region).getByText("Saved converters: The saved converters couldn't be listed: permission denied"))
+      .toBeInTheDocument()
+    expect(region).toHaveAttribute('tabindex', '0')
+  })
+
+  it('should show no list when every saved entry was restored', async () => {
+    api.getRuntimeStatus.mockResolvedValue({ ...ready, restore_failures: [] })
+    render(
+      <TestWrapper>
+        <Reinitialize version="saved-v1" hasUnsavedChanges={false} liveReinitializationEnabled />
+      </TestWrapper>,
+    )
+
+    await waitFor(() => expect(screen.getByRole('button', { name: 'Reinitialize PyRIT' })).toBeEnabled())
+    expect(screen.queryByRole('region', { name: "Saved entries that couldn't be restored" })).not.toBeInTheDocument()
+  })
+
   describe('runtime status polling', () => {
     beforeEach(() => {
       jest.useFakeTimers()

frontend/src/components/Configuration/Reinitialize.tsx

@@ -13,6 +13,7 @@ import { toApiError } from '@/services/errors'
 import type { RuntimeStatus } from '@/types'
 
 import { useReinitializeStyles } from './Reinitialize.styles'
+import RestoreFailures from './RestoreFailures'
 
 interface ReinitializeProps {
   version: string
@@ -81,13 +82,17 @@ export default function Reinitialize({
   return (
     <section aria-label="Reinitialize PyRIT" className={styles.root}>
       {status && (showRuntimeStatus || statusIsError || !status.enabled) && (
-        <MessageBar intent={statusIsError ? 'error' : 'info'}>
+        // Multiline, since a status message such as a restore failure can be long and change after the bar lays out.
+        <MessageBar intent={statusIsError ? 'error' : 'info'} layout="multiline">
           <MessageBarBody>
             Runtime: {status.state}. {status.message}
             {!status.enabled && ' Reinitialization requires one backend worker and one replica.'}
           </MessageBarBody>
         </MessageBar>
       )}
+      {status?.restore_failures && status.restore_failures.length > 0 && (
+        <RestoreFailures failures={status.restore_failures} />
+      )}
       {pollError && <MessageBar intent="error"><MessageBarBody>{pollError}</MessageBarBody></MessageBar>}
       {applyError && <MessageBar intent="error"><MessageBarBody>{applyError}</MessageBarBody></MessageBar>}
       {hasUnsavedChanges && <Text>Save or explicitly discard unsaved edits before reinitializing.</Text>}

frontend/src/components/Configuration/RestoreFailures.styles.ts

@@ -0,0 +1,15 @@
+import { makeStyles } from '@fluentui/react-components'
+
+export const useRestoreFailuresStyles = makeStyles({
+  // The page doesn't scroll past the editor, so a long list scrolls on its own. It fills its column, and a long
+  // name or message wraps instead of widening the list past it.
+  list: {
+    alignSelf: 'stretch',
+    boxSizing: 'border-box',
+    maxWidth: '100%',
+    minWidth: 0,
+    maxHeight: '30vh',
+    overflowWrap: 'anywhere',
+    overflowY: 'auto',
+  },
+})

frontend/src/components/Configuration/RestoreFailures.tsx

@@ -0,0 +1,42 @@
+import type { RegistryRecipeKind, RestoreFailure } from '@/types'
+
+import { useRestoreFailuresStyles } from './RestoreFailures.styles'
+
+const KIND_LABELS: Record<RegistryRecipeKind, string> = {
+  targets: 'Target',
+  converters: 'Converter',
+  scorers: 'Scorer',
+  techniques: 'Technique',
+}
+
+function describeFailure(failure: RestoreFailure): string {
+  const subject = failure.name === null ? `Saved ${failure.kind}` : `${KIND_LABELS[failure.kind]} ${failure.name}`
+  // The message usually names the entry it depends on already.
+  const dependency = failure.dependency && !failure.message.includes(`'${failure.dependency.name}'`)
+    ? ` (${KIND_LABELS[failure.dependency.kind].toLowerCase()} ${failure.dependency.name})`
+    : ''
+  return `${subject}: ${failure.message}${dependency}`
+}
+
+interface RestoreFailuresProps {
+  failures: RestoreFailure[]
+}
+
+export default function RestoreFailures({ failures }: RestoreFailuresProps) {
+  const styles = useRestoreFailuresStyles()
+  return (
+    <div
+      role="region"
+      aria-label="Saved entries that couldn't be restored"
+      // Focusable so a long list can be scrolled from the keyboard.
+      tabIndex={0}
+      className={styles.list}
+    >
+      <ul>
+        {failures.map((failure) => (
+          <li key={`${failure.kind}/${failure.name ?? ''}`}>{describeFailure(failure)}</li>
+        ))}
+      </ul>
+    </div>
+  )
+}

frontend/src/components/Layout/MainLayout.styles.ts

@@ -33,6 +33,9 @@ export const useMainLayoutStyles = makeStyles({
       transform: 'translateY(0)',
     },
   },
+  banner: {
+    flexShrink: 0,
+  },
   topBar: {
     minHeight: '60px',
     flexShrink: 0,

frontend/src/components/Layout/MainLayout.test.tsx

@@ -77,6 +77,23 @@ describe("MainLayout", () => {
     expect(screen.getByRole("button", { name: "Collapse sidebar" })).toHaveAttribute("aria-expanded", "true");
   });
 
+  it("shows a banner between the skip link and the top bar, in the layout column", async () => {
+    mockedVersionApi.getVersion.mockResolvedValue({ version: "1.0.0" });
+
+    renderWithProvider(
+      <MainLayout {...defaultProps} banner={<div role="status">Saved entries need attention</div>}>
+        <div>Child Content</div>
+      </MainLayout>
+    );
+
+    const skipLink = screen.getByRole("link", { name: "Skip to main content" });
+    const banner = screen.getByRole("status");
+    const subtitle = screen.getByText("Python Risk Identification Tool");
+    expect(skipLink.compareDocumentPosition(banner) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
+    expect(banner.compareDocumentPosition(subtitle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
+    expect(screen.getByRole("main")).not.toContainElement(banner);
+  });
+
   it("renders the header with title and subtitle", async () => {
     mockedVersionApi.getVersion.mockResolvedValue({ version: "1.0.0" });
 

frontend/src/components/Layout/MainLayout.tsx

@@ -35,6 +35,8 @@ interface MainLayoutProps extends OperationControlProps {
   onStartTour?: () => void
   labelsBusy?: boolean
   featureNavigation?: React.ReactNode
+  /** Shown above the top bar, so the content below gets smaller instead of losing its bottom. */
+  banner?: React.ReactNode
 }
 
 export default function MainLayout({
@@ -53,6 +55,7 @@ export default function MainLayout({
   onStartTour,
   labelsBusy = false,
   featureNavigation,
+  banner,
 }: MainLayoutProps) {
   usePreventFileNavigation()
   const styles = useMainLayoutStyles()
@@ -89,6 +92,7 @@ export default function MainLayout({
       <a href="#main-content" className={styles.skipLink}>
         Skip to main content
       </a>
+      {banner && <div className={styles.banner}>{banner}</div>}
       <div className={styles.topBar}>
         <Tooltip
           content={