Keep search working when an app is retired, and list every app

app/api/self-host-apps/search/route.ts

@@ -1,5 +1,5 @@
 import { NextResponse } from "next/server";
-import { getSelfHostAppsSearch } from "@/lib/self-host-apps-search-data";
+import { getSelfHostAppsSearchForApi } from "@/lib/self-host-apps-search-data";
 import { getPublishedPage } from "@/lib/page-registry";
 import {
   parseSelfHostAppsSearchParams, SelfHostAppsSearchInputError,
@@ -14,9 +14,12 @@ export async function GET(request: Request) {
     if (!await getPublishedPage("self-host-apps")) {
       return NextResponse.json({ error: "Directory not found." }, { status: 404, headers: noStore });
     }
-    const response = await getSelfHostAppsSearch(params);
+    const { response, complete } = await getSelfHostAppsSearchForApi(params);
+    // An incomplete result is correct only until the cache refresh lands, so it must not be stored.
     return NextResponse.json(response, {
-      headers: { "Cache-Control": "public, max-age=0, must-revalidate, s-maxage=604800", "Cache-Tag": "stack127-public,page:api:self-host-apps:search,self-host-catalog" },
+      headers: complete
+        ? { "Cache-Control": "public, max-age=0, must-revalidate, s-maxage=604800", "Cache-Tag": "stack127-public,page:api:self-host-apps:search,self-host-catalog" }
+        : noStore,
     });
   } catch (error) {
     if (error instanceof SelfHostAppsSearchInputError) {

lib/self-host-apps-search-data.ts

@@ -11,21 +11,37 @@ async function search(
   params: SelfHostAppsSearchParams,
   loadIndex: typeof getDirectoryAppIndex,
   loadDetails: typeof getDirectoryAppsBySlugs,
-): Promise<SelfHostAppsSearchResponse> {
+  missing: "fail" | "drop" = "fail",
+): Promise<{ response: SelfHostAppsSearchResponse; complete: boolean }> {
   const index = await loadIndex();
   const results = selectSelfHostApps(index, params);
   const details = await loadDetails(results.apps.map(app => app.slug));
   const bySlug = new Map(details.map(app => [app.slug, app]));
-  const detailApps = results.apps.map(app => {
-    const detail = bySlug.get(app.slug);
-    if (!detail) throw new Error(`Missing search detail: ${app.slug}`);
-    return detail;
+  const apps = results.apps.filter(app => {
+    if (bySlug.has(app.slug)) return true;
+    if (missing === "fail") throw new Error(`Missing search detail: ${app.slug}`);
+    return false;
   });
-  return { ...results, detailApps, facets: selfHostAppsFacets(index, params.category) };
+  // A page with nothing left would look like "no matches" and hide the other pages, so fail and let the reader retry.
+  if (results.apps.length > 0 && apps.length === 0) throw new Error("Missing search details for every app on the page.");
+  const detailApps = apps.map(app => bySlug.get(app.slug)!);
+  return {
+    response: { ...results, apps, detailApps, facets: selfHostAppsFacets(index, params.category) },
+    complete: apps.length === results.apps.length,
+  };
 }
 
 export const getSelfHostAppsSearch = (params: SelfHostAppsSearchParams) =>
-  search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs);
+  search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs).then(result => result.response);
+
+/**
+ * For the public API. An app retired moments ago can still be in the cached
+ * index while its details are gone. The API leaves that app out and reports
+ * the result as incomplete, so one retired app does not fail every search
+ * that would have shown it. Pages and cache verification still fail loudly.
+ */
+export const getSelfHostAppsSearchForApi = (params: SelfHostAppsSearchParams) =>
+  search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs, "drop");
 
 export const loadSelfHostAppsSearch = (params: SelfHostAppsSearchParams) =>
-  search(params, loadDirectoryAppIndex, loadDirectoryAppsBySlugs);
+  search(params, loadDirectoryAppIndex, loadDirectoryAppsBySlugs).then(result => result.response);

scripts/self-host-apps/ovh/README.md

@@ -61,10 +61,14 @@ Node, never evaluated as shell code. Credentials:
   `sslmode=disable`; use the same connection as launch-directory metrics.
 - `GITHUB_TOKEN`: GitHub discovery, catalogs, facts and technical jobs. Copy an
   existing authorised Infisical credential; no interactive `gh` login is needed.
-- `AWESOME_LIST_GITHUB_TOKEN`: required only for `self-host-awesome-list`. Add it
-  to Infisical `stack127 /vps/self-host-apps`. Use a fine-grained token with
-  resource owner `stack127labs`, repository access only to
-  `awesome-self-hosted-apps`, and Contents read/write (Metadata read is automatic).
+- `PUBLIC_REPOS_GITHUB_TOKEN`: required only for `self-host-awesome-list`. Add it
+  to Infisical `stack127 /vps/self-host-apps`. It is one shared token for every
+  public output repository that a server job pushes to. Use a fine-grained token
+  with resource owner `stack127labs`, "Only select repositories", and Contents
+  read/write (Metadata read is automatic). Select `awesome-self-hosted-apps`
+  now; when a new list or dataset repository gets a job, add that repository to
+  the same token on GitHub, with no new key. Never select the main `stack127`
+  repository: a push there deploys to production.
   The existing `GITHUB_TOKEN` is for discovery/API reads; its write access is not
   assumed. Git uses a temporary askpass script that reads the token from the
   environment; no token is stored in a remote URL or Git configuration.
@@ -132,7 +136,7 @@ never writes inferred editorial technical facts.
 ## Install after merge
 
 The lead installs cron only after the reviewed code reaches main. First ensure
-`DATABASE_URL`, `GITHUB_TOKEN`, and `AWESOME_LIST_GITHUB_TOKEN` exist and the Infisical sync succeeds.
+`DATABASE_URL`, `GITHUB_TOKEN`, and `PUBLIC_REPOS_GITHUB_TOKEN` exist and the Infisical sync succeeds.
 `GITLAB_TOKEN` and `CODEBERG_TOKEN` are optional. If the clone does not
 exist, create it as ubuntu (sudo may be needed just for the directory):
 
@@ -234,7 +238,7 @@ file belongs inside either checkout.
 ## Weekly public list: install and operate
 
 Built, waiting for install. After the reviewed merge, the lead adds
-`AWESOME_LIST_GITHUB_TOKEN` to Infisical as described above and waits for the
+`PUBLIC_REPOS_GITHUB_TOKEN` to Infisical as described above and waits for the
 sync to deliver the mode-600 env file. Do not edit that file on OVH. The existing
 dataset sync uses `gh` with `GH_TOKEN`/`GITHUB_TOKEN`; this job uses Git HTTPS
 with its own token and does not need an interactive login or `gh`.

scripts/self-host-apps/ovh/run-job.sh

@@ -264,7 +264,7 @@ execute() {
   }
   # Use Node's dotenv parser; never evaluate secret values as shell code.
   export GITHUB_STEP_SUMMARY="$report_dir/$run_id-summary.md"
-  unset DATABASE_URL GITHUB_TOKEN GITLAB_TOKEN CODEBERG_TOKEN AWESOME_LIST_GITHUB_TOKEN
+  unset DATABASE_URL GITHUB_TOKEN GITLAB_TOKEN CODEBERG_TOKEN PUBLIC_REPOS_GITHUB_TOKEN
   local started
   started=$(date +%s%3N)
   # Run the updated step definitions. Syntax/launcher failures reach the stable

scripts/self-host-apps/ovh/validate-env.mjs

@@ -6,7 +6,7 @@ let database;
 try { database = new URL(process.env.DATABASE_URL); } catch { fail('DATABASE_URL is missing or invalid'); }
 if (database.hostname !== '127.0.0.1' || database.username !== 'stack127_app' || database.pathname !== '/postgres' || (database.port || '5432') !== '5432' || database.searchParams.get('sslmode') !== 'disable') fail('DATABASE_URL must use the loopback production app connection with sslmode=disable');
 if (['self-host-discovery-github', 'self-host-discovery-catalogs', 'self-host-refresh-github', 'self-host-refresh-technical'].includes(job) && !process.env.GITHUB_TOKEN?.trim()) fail('GITHUB_TOKEN is missing');
-if (job === 'self-host-awesome-list' && !process.env.AWESOME_LIST_GITHUB_TOKEN?.trim()) fail('AWESOME_LIST_GITHUB_TOKEN is missing');
+if (job === 'self-host-awesome-list' && !process.env.PUBLIC_REPOS_GITHUB_TOKEN?.trim()) fail('PUBLIC_REPOS_GITHUB_TOKEN is missing');
 // GITLAB_TOKEN and CODEBERG_TOKEN are optional: both APIs answer public reads without them.
 // Descriptor 8 holds the checkout flock. The step shell must retain it if this launcher dies.
 const child = spawn('/bin/bash', [wrapper, '--steps', job, checkout, runId], { stdio: ['inherit', 'inherit', 'inherit', 'ignore', 'ignore', 'ignore', 'ignore', 'ignore', 8] });

scripts/stack127/repos/export-awesome-list.mjs

@@ -115,9 +115,7 @@ export function normalizeApp(row) {
   };
 }
 
-// The directory can hold one project twice, for example an appliance repackage
-// such as "binhex-jellyfin" beside "Jellyfin". The list shows each source
-// repository once and keeps the plainest entry.
+// Used to report entries that share a source repository; the list keeps all of them.
 // Host and path identify a repository; scheme, query, fragment, a trailing
 // slash, and ".git" do not. Paths are case-insensitive only on these hosts.
 const caseInsensitiveHosts = new Set(["github.com", "gitlab.com", "codeberg.org"]);
@@ -127,20 +125,9 @@ export function repositoryKey(value) {
   const repoPath = url.pathname.replace(/\/+$/, "").replace(/\.git$/, "");
   return `${host}${caseInsensitiveHosts.has(url.hostname.replace(/^www\./, "")) ? repoPath.toLowerCase() : repoPath}`;
 }
-const repackaged = (name) => /^binhex-|-for-/i.test(name);
-
-export function onePerRepository(apps) {
-  const best = new Map();
-  const better = (a, b) => Number(repackaged(a.name)) - Number(repackaged(b.name))
-    || Number(!a.website_url) - Number(!b.website_url)
-    || a.name.length - b.name.length
-    || a.slug.localeCompare(b.slug);
-  for (const app of apps) {
-    if (!app.source_code_url) continue;
-    const key = repositoryKey(app.source_code_url);
-    if (!best.has(key) || better(app, best.get(key)) < 0) best.set(key, app);
-  }
-  return apps.filter((app) => !app.source_code_url || best.get(repositoryKey(app.source_code_url)) === app);
+export function repeatedRepositoryEntries(apps) {
+  const keys = apps.filter((app) => app.source_code_url).map((app) => repositoryKey(app.source_code_url));
+  return keys.length - new Set(keys).size;
 }
 
 function appRow(app) {
@@ -247,11 +234,11 @@ async function main() {
   });
   if (!categories.length || !apps.length) throw new Error("No published categories or apps to export");
 
-  // Drop unpublished categories first, so the entry kept for a repository is one that can be listed.
   const appsByCategory = new Map(categories.map((category) => [category.slug, []]));
   for (const app of apps) app.categories = app.categories.filter((slug) => appsByCategory.has(slug));
   const categorized = apps.filter((app) => app.categories.length);
-  const listed = onePerRepository(categorized);
+  // Every published app is listed. The directory retires duplicates; apps that share a repository here are separate apps.
+  const listed = categorized;
   for (const app of listed) for (const slug of app.categories) appsByCategory.get(slug).push(app);
   const date = new Date().toISOString().slice(0, 10);
 
@@ -264,7 +251,7 @@ async function main() {
   writeText(path.join(outDir, "data", "apps.csv"), toCsv(listed.map((app) => ({ ...app, categories: app.categories.join("|") })), csvColumns));
   writeText(path.join(outDir, "data", "apps.json"), JSON.stringify({ generated: date, source: `${site}/self-host-apps`, apps: listed }, null, 1));
 
-  console.log(JSON.stringify({ out: outDir, apps: listed.length, repeatedRepositoryEntries: categorized.length - listed.length, withoutCategory: apps.length - categorized.length, categories: categories.length }));
+  console.log(JSON.stringify({ out: outDir, apps: listed.length, repeatedRepositoryEntries: repeatedRepositoryEntries(listed), withoutCategory: apps.length - categorized.length, categories: categories.length }));
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {

scripts/stack127/repos/sync-awesome-list.mjs

@@ -69,7 +69,7 @@ export function syncGenerated(source, destination, next = inspectExport(source))
   for (const name of old) if (!next.files.includes(`categories/${name}`)) rmSync(path.join(destination, 'categories', name));
 }
 function redact(text) {
-  for (const value of [process.env.AWESOME_LIST_GITHUB_TOKEN, process.env.DATABASE_URL]) if (value) text = text.replaceAll(value, '[redacted]');
+  for (const value of [process.env.PUBLIC_REPOS_GITHUB_TOKEN, process.env.DATABASE_URL]) if (value) text = text.replaceAll(value, '[redacted]');
   return text;
 }
 export function publishExport(source, destination, { git, allowCountDrop = false } = {}) {
@@ -88,15 +88,15 @@ export function publishExport(source, destination, { git, allowCountDrop = false
   return true;
 }
 export function runWeekly({ workRoot = '/srv/stack127-data/self-host-apps', allowCountDrop = process.env.AWESOME_LIST_ALLOW_COUNT_DROP === '1' } = {}) {
-  if (!process.env.AWESOME_LIST_GITHUB_TOKEN?.trim()) throw new Error('AWESOME_LIST_GITHUB_TOKEN is missing');
+  if (!process.env.PUBLIC_REPOS_GITHUB_TOKEN?.trim()) throw new Error('PUBLIC_REPOS_GITHUB_TOKEN is missing');
   safePath(workRoot);
   mkdirSync(workRoot, { recursive: true });
   const work = mkdtempSync(path.join(workRoot, 'awesome-list-'));
   const source = path.join(work, 'export');
   const destination = path.join(work, 'repo');
   const askpass = path.join(work, 'askpass.sh');
   // Only the variable name goes on disk. The token is never in args/config/remote.
-  writeFileSync(askpass, '#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" x-access-token ;; *) printf "%s\\n" "$AWESOME_LIST_GITHUB_TOKEN" ;; esac\n', { mode: 0o700 });
+  writeFileSync(askpass, '#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" x-access-token ;; *) printf "%s\\n" "$PUBLIC_REPOS_GITHUB_TOKEN" ;; esac\n', { mode: 0o700 });
   const env = { ...process.env, GIT_AUTHOR_NAME: 'stack127 list bot', GIT_AUTHOR_EMAIL: 'awesome-list-bot@users.noreply.github.com', GIT_COMMITTER_NAME: 'stack127 list bot', GIT_COMMITTER_EMAIL: 'awesome-list-bot@users.noreply.github.com', GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: askpass, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' };
   const run = (command, args, cwd) => {
     const result = spawnSync(command, args, { cwd, env, encoding: 'utf8', timeout: 300000, maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] });

stack127_docs/brand-growth/status.md

@@ -28,25 +28,23 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h
 - 2026-11-13: all 54 published category pages (42 self-host, 12 CLI) rebuilt with the main page's cards, filters, search, sort, and pagination, plus a "how to choose" section below the cards. Released in pull request #120, commit `070a5c79`.
 - 2026-11-13: 98 duplicate directory entries retired in production (4,913 to 4,815 published self-hostable apps). The decisions are in `scripts/self-host-apps/batches/2026-11-13-published-duplicates-reviewed.json`. The public list was refreshed by hand: 4,803 apps in 42 categories.
 - 2026-11-13: owner did the "Request indexing" clicks for the 10 hub URLs.
+- 2026-11-13: retiring the 98 apps broke live search for about 20 minutes: `/api/self-host-apps/search` returned 503 for any result that included a retired app, until the frontend refresh run finished. Fixed the same day: the API now leaves out an app whose details are gone and does not cache that response. The public list now shows every published app, not one per repository.
 
 ## Next
 
 | Step | Who | State |
 | --- | --- | --- |
 | Request indexing for the 10 largest self-host category pages | Owner | Open |
 | Submit the remaining 148 sitemap URLs to Bing, 100 a day | Agent | Open |
-| Install the weekly list refresh on OVH (`self-host-awesome-list`, Monday 14:02 UTC) | Agent | Code is on `main`. Needs `AWESOME_LIST_GITHUB_TOKEN` in Infisical `stack127 /vps/self-host-apps`: a fine-grained token limited to that repository with Contents read and write. Then follow `scripts/self-host-apps/ovh/README.md`, including the supervisor reinstall |
-| Make search survive a retired app | Agent, after the owner's go | Open. See the gap below |
+| Install the weekly list refresh on OVH (`self-host-awesome-list`, Monday 14:02 UTC) | Agent | Code is on `main`. Needs `PUBLIC_REPOS_GITHUB_TOKEN` in Infisical `stack127 /vps/self-host-apps`: one shared fine-grained token for the public output repositories (select repositories, Contents read and write), never the main repository. Then follow `scripts/self-host-apps/ovh/README.md`, including the supervisor reinstall |
 | Pull the numbers and write the first data story | Agent | Open. Use 4,815 apps |
 | Post the story on r/selfhosted | Owner | After review |
 | Measure referring domains as a baseline | Agent | Open |
 | CLI apps list, then ad networks list | Agent | After the first list |
 
 ## Known data gaps
 
-- Retiring apps broke live search for about 20 minutes on 2026-11-13. `/api/self-host-apps/search` returned 503 for any result that included a retired app, because the cached app index still listed it while its details were gone. It recovered when the frontend refresh run finished. `lib/self-host-apps-search-data.ts` throws "Missing search detail"; it should drop such an app from the result.
 - 11 possible duplicate groups were left untouched as unsure, for example "Borg" and "borgserver", and "ONLYOFFICE" and "ONLYOFFICE Docs". They are marked `unsure` in the batch file.
-- The public list still shows one app per source repository, so 12 real apps that share a repository with another app, for example "Joplin Server", are missing from it (4,803 against 4,815). Now that the directory is clean, the export can list every published app.
 - The "Has mobile app" filter matches 0 to 4 apps per category in development, and "Needs database" is empty for most apps. Both need data work.
 - Two CLI categories, `communication` and `games-fun`, are below 15 apps and unpublished. Their intros are placeholder text.
 - The main repository `stack127labs/stack127` is public on GitHub for build-minute reasons. The owner plans to make it private later.
@@ -55,4 +53,3 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h
 ## Waiting on the owner
 
 - The GitHub token for the weekly list refresh.
-- A yes to release the search fix.

stack127_docs/stack127-repos.md

@@ -102,7 +102,7 @@ npm run stack127:repo:tool:new -- --slug port-checker --name "Port Checker" --dr
 
 Dataset exports read verified production data, pick public columns, and sort rows the same way every time. They show a dry-run diff before any reviewed write or push.
 
-The awesome-list export builds the files of `stack127labs/awesome-self-hosted-apps`: a README with a category index, one Markdown table per published category page, and `data/apps.csv` and `data/apps.json`. It lists each source repository once, so its app count is lower than the directory's. The OVH `self-host-awesome-list` job refreshes the public repository on Mondays at 14:02 UTC, with hourly catch-up. It exports into a fresh directory outside the code checkout, validates the files, refuses app-count drops over 10% and any category-count drop, then commits only changed generated files and pushes to `main`. Each run uses a fresh clone outside the code checkout, so interrupted commits and failed pushes cannot leave pending local history for the next run. Unrelated repository files stay intact. Setup, the repository-only token, and the manual count override are documented in [the OVH runner guide](../scripts/self-host-apps/ovh/README.md). Built, waiting for install.
+The awesome-list export builds the files of `stack127labs/awesome-self-hosted-apps`: a README with a category index, one Markdown table per published category page, and `data/apps.csv` and `data/apps.json`. It lists every published app that has a published category page. The OVH `self-host-awesome-list` job refreshes the public repository on Mondays at 14:02 UTC, with hourly catch-up. It exports into a fresh directory outside the code checkout, validates the files, refuses app-count drops over 10% and any category-count drop, then commits only changed generated files and pushes to `main`. Each run uses a fresh clone outside the code checkout, so interrupted commits and failed pushes cannot leave pending local history for the next run. Unrelated repository files stay intact. Setup, the repository-only token, and the manual count override are documented in [the OVH runner guide](../scripts/self-host-apps/ovh/README.md). Built, waiting for install.
 
 Exports must use `.env.database`, an explicit production `--database-url` where supported, or the awesome-list export’s `--database-env` option with the OVH runner’s validated production `DATABASE_URL`. They never fall back to `.env.local`.