diff --git a/app/api/self-host-apps/search/route.ts b/app/api/self-host-apps/search/route.ts index f43f4d60..70d52b57 100644 --- a/app/api/self-host-apps/search/route.ts +++ b/app/api/self-host-apps/search/route.ts @@ -1,5 +1,5 @@ import { NextResponse } from "next/server"; -import { getSelfHostAppsSearch } from "@/lib/self-host-apps-search-data"; +import { getSelfHostAppsSearchForApi } from "@/lib/self-host-apps-search-data"; import { getPublishedPage } from "@/lib/page-registry"; import { parseSelfHostAppsSearchParams, SelfHostAppsSearchInputError, @@ -14,9 +14,12 @@ export async function GET(request: Request) { if (!await getPublishedPage("self-host-apps")) { return NextResponse.json({ error: "Directory not found." }, { status: 404, headers: noStore }); } - const response = await getSelfHostAppsSearch(params); + const { response, complete } = await getSelfHostAppsSearchForApi(params); + // An incomplete result is correct only until the cache refresh lands, so it must not be stored. return NextResponse.json(response, { - headers: { "Cache-Control": "public, max-age=0, must-revalidate, s-maxage=604800", "Cache-Tag": "stack127-public,page:api:self-host-apps:search,self-host-catalog" }, + headers: complete + ? { "Cache-Control": "public, max-age=0, must-revalidate, s-maxage=604800", "Cache-Tag": "stack127-public,page:api:self-host-apps:search,self-host-catalog" } + : noStore, }); } catch (error) { if (error instanceof SelfHostAppsSearchInputError) { diff --git a/lib/self-host-apps-search-data.ts b/lib/self-host-apps-search-data.ts index b5f23085..1643d988 100644 --- a/lib/self-host-apps-search-data.ts +++ b/lib/self-host-apps-search-data.ts @@ -11,21 +11,37 @@ async function search( params: SelfHostAppsSearchParams, loadIndex: typeof getDirectoryAppIndex, loadDetails: typeof getDirectoryAppsBySlugs, -): Promise { + missing: "fail" | "drop" = "fail", +): Promise<{ response: SelfHostAppsSearchResponse; complete: boolean }> { const index = await loadIndex(); const results = selectSelfHostApps(index, params); const details = await loadDetails(results.apps.map(app => app.slug)); const bySlug = new Map(details.map(app => [app.slug, app])); - const detailApps = results.apps.map(app => { - const detail = bySlug.get(app.slug); - if (!detail) throw new Error(`Missing search detail: ${app.slug}`); - return detail; + const apps = results.apps.filter(app => { + if (bySlug.has(app.slug)) return true; + if (missing === "fail") throw new Error(`Missing search detail: ${app.slug}`); + return false; }); - return { ...results, detailApps, facets: selfHostAppsFacets(index, params.category) }; + // A page with nothing left would look like "no matches" and hide the other pages, so fail and let the reader retry. + if (results.apps.length > 0 && apps.length === 0) throw new Error("Missing search details for every app on the page."); + const detailApps = apps.map(app => bySlug.get(app.slug)!); + return { + response: { ...results, apps, detailApps, facets: selfHostAppsFacets(index, params.category) }, + complete: apps.length === results.apps.length, + }; } export const getSelfHostAppsSearch = (params: SelfHostAppsSearchParams) => - search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs); + search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs).then(result => result.response); + +/** + * For the public API. An app retired moments ago can still be in the cached + * index while its details are gone. The API leaves that app out and reports + * the result as incomplete, so one retired app does not fail every search + * that would have shown it. Pages and cache verification still fail loudly. + */ +export const getSelfHostAppsSearchForApi = (params: SelfHostAppsSearchParams) => + search(params, getDirectoryAppIndex, getDirectoryAppsBySlugs, "drop"); export const loadSelfHostAppsSearch = (params: SelfHostAppsSearchParams) => - search(params, loadDirectoryAppIndex, loadDirectoryAppsBySlugs); + search(params, loadDirectoryAppIndex, loadDirectoryAppsBySlugs).then(result => result.response); diff --git a/scripts/self-host-apps/ovh/README.md b/scripts/self-host-apps/ovh/README.md index 4d4485a5..daefd3cf 100644 --- a/scripts/self-host-apps/ovh/README.md +++ b/scripts/self-host-apps/ovh/README.md @@ -61,10 +61,14 @@ Node, never evaluated as shell code. Credentials: `sslmode=disable`; use the same connection as launch-directory metrics. - `GITHUB_TOKEN`: GitHub discovery, catalogs, facts and technical jobs. Copy an existing authorised Infisical credential; no interactive `gh` login is needed. -- `AWESOME_LIST_GITHUB_TOKEN`: required only for `self-host-awesome-list`. Add it - to Infisical `stack127 /vps/self-host-apps`. Use a fine-grained token with - resource owner `stack127labs`, repository access only to - `awesome-self-hosted-apps`, and Contents read/write (Metadata read is automatic). +- `PUBLIC_REPOS_GITHUB_TOKEN`: required only for `self-host-awesome-list`. Add it + to Infisical `stack127 /vps/self-host-apps`. It is one shared token for every + public output repository that a server job pushes to. Use a fine-grained token + with resource owner `stack127labs`, "Only select repositories", and Contents + read/write (Metadata read is automatic). Select `awesome-self-hosted-apps` + now; when a new list or dataset repository gets a job, add that repository to + the same token on GitHub, with no new key. Never select the main `stack127` + repository: a push there deploys to production. The existing `GITHUB_TOKEN` is for discovery/API reads; its write access is not assumed. Git uses a temporary askpass script that reads the token from the environment; no token is stored in a remote URL or Git configuration. @@ -132,7 +136,7 @@ never writes inferred editorial technical facts. ## Install after merge The lead installs cron only after the reviewed code reaches main. First ensure -`DATABASE_URL`, `GITHUB_TOKEN`, and `AWESOME_LIST_GITHUB_TOKEN` exist and the Infisical sync succeeds. +`DATABASE_URL`, `GITHUB_TOKEN`, and `PUBLIC_REPOS_GITHUB_TOKEN` exist and the Infisical sync succeeds. `GITLAB_TOKEN` and `CODEBERG_TOKEN` are optional. If the clone does not exist, create it as ubuntu (sudo may be needed just for the directory): @@ -234,7 +238,7 @@ file belongs inside either checkout. ## Weekly public list: install and operate Built, waiting for install. After the reviewed merge, the lead adds -`AWESOME_LIST_GITHUB_TOKEN` to Infisical as described above and waits for the +`PUBLIC_REPOS_GITHUB_TOKEN` to Infisical as described above and waits for the sync to deliver the mode-600 env file. Do not edit that file on OVH. The existing dataset sync uses `gh` with `GH_TOKEN`/`GITHUB_TOKEN`; this job uses Git HTTPS with its own token and does not need an interactive login or `gh`. diff --git a/scripts/self-host-apps/ovh/run-job.sh b/scripts/self-host-apps/ovh/run-job.sh index 1690eb88..1262f9f1 100755 --- a/scripts/self-host-apps/ovh/run-job.sh +++ b/scripts/self-host-apps/ovh/run-job.sh @@ -264,7 +264,7 @@ execute() { } # Use Node's dotenv parser; never evaluate secret values as shell code. export GITHUB_STEP_SUMMARY="$report_dir/$run_id-summary.md" - unset DATABASE_URL GITHUB_TOKEN GITLAB_TOKEN CODEBERG_TOKEN AWESOME_LIST_GITHUB_TOKEN + unset DATABASE_URL GITHUB_TOKEN GITLAB_TOKEN CODEBERG_TOKEN PUBLIC_REPOS_GITHUB_TOKEN local started started=$(date +%s%3N) # Run the updated step definitions. Syntax/launcher failures reach the stable diff --git a/scripts/self-host-apps/ovh/validate-env.mjs b/scripts/self-host-apps/ovh/validate-env.mjs index 419ed159..101b51ac 100644 --- a/scripts/self-host-apps/ovh/validate-env.mjs +++ b/scripts/self-host-apps/ovh/validate-env.mjs @@ -6,7 +6,7 @@ let database; try { database = new URL(process.env.DATABASE_URL); } catch { fail('DATABASE_URL is missing or invalid'); } if (database.hostname !== '127.0.0.1' || database.username !== 'stack127_app' || database.pathname !== '/postgres' || (database.port || '5432') !== '5432' || database.searchParams.get('sslmode') !== 'disable') fail('DATABASE_URL must use the loopback production app connection with sslmode=disable'); if (['self-host-discovery-github', 'self-host-discovery-catalogs', 'self-host-refresh-github', 'self-host-refresh-technical'].includes(job) && !process.env.GITHUB_TOKEN?.trim()) fail('GITHUB_TOKEN is missing'); -if (job === 'self-host-awesome-list' && !process.env.AWESOME_LIST_GITHUB_TOKEN?.trim()) fail('AWESOME_LIST_GITHUB_TOKEN is missing'); +if (job === 'self-host-awesome-list' && !process.env.PUBLIC_REPOS_GITHUB_TOKEN?.trim()) fail('PUBLIC_REPOS_GITHUB_TOKEN is missing'); // GITLAB_TOKEN and CODEBERG_TOKEN are optional: both APIs answer public reads without them. // Descriptor 8 holds the checkout flock. The step shell must retain it if this launcher dies. const child = spawn('/bin/bash', [wrapper, '--steps', job, checkout, runId], { stdio: ['inherit', 'inherit', 'inherit', 'ignore', 'ignore', 'ignore', 'ignore', 'ignore', 8] }); diff --git a/scripts/stack127/repos/export-awesome-list.mjs b/scripts/stack127/repos/export-awesome-list.mjs index 0db07ae0..b13c1342 100644 --- a/scripts/stack127/repos/export-awesome-list.mjs +++ b/scripts/stack127/repos/export-awesome-list.mjs @@ -115,9 +115,7 @@ export function normalizeApp(row) { }; } -// The directory can hold one project twice, for example an appliance repackage -// such as "binhex-jellyfin" beside "Jellyfin". The list shows each source -// repository once and keeps the plainest entry. +// Used to report entries that share a source repository; the list keeps all of them. // Host and path identify a repository; scheme, query, fragment, a trailing // slash, and ".git" do not. Paths are case-insensitive only on these hosts. const caseInsensitiveHosts = new Set(["github.com", "gitlab.com", "codeberg.org"]); @@ -127,20 +125,9 @@ export function repositoryKey(value) { const repoPath = url.pathname.replace(/\/+$/, "").replace(/\.git$/, ""); return `${host}${caseInsensitiveHosts.has(url.hostname.replace(/^www\./, "")) ? repoPath.toLowerCase() : repoPath}`; } -const repackaged = (name) => /^binhex-|-for-/i.test(name); - -export function onePerRepository(apps) { - const best = new Map(); - const better = (a, b) => Number(repackaged(a.name)) - Number(repackaged(b.name)) - || Number(!a.website_url) - Number(!b.website_url) - || a.name.length - b.name.length - || a.slug.localeCompare(b.slug); - for (const app of apps) { - if (!app.source_code_url) continue; - const key = repositoryKey(app.source_code_url); - if (!best.has(key) || better(app, best.get(key)) < 0) best.set(key, app); - } - return apps.filter((app) => !app.source_code_url || best.get(repositoryKey(app.source_code_url)) === app); +export function repeatedRepositoryEntries(apps) { + const keys = apps.filter((app) => app.source_code_url).map((app) => repositoryKey(app.source_code_url)); + return keys.length - new Set(keys).size; } function appRow(app) { @@ -247,11 +234,11 @@ async function main() { }); if (!categories.length || !apps.length) throw new Error("No published categories or apps to export"); - // Drop unpublished categories first, so the entry kept for a repository is one that can be listed. const appsByCategory = new Map(categories.map((category) => [category.slug, []])); for (const app of apps) app.categories = app.categories.filter((slug) => appsByCategory.has(slug)); const categorized = apps.filter((app) => app.categories.length); - const listed = onePerRepository(categorized); + // Every published app is listed. The directory retires duplicates; apps that share a repository here are separate apps. + const listed = categorized; for (const app of listed) for (const slug of app.categories) appsByCategory.get(slug).push(app); const date = new Date().toISOString().slice(0, 10); @@ -264,7 +251,7 @@ async function main() { writeText(path.join(outDir, "data", "apps.csv"), toCsv(listed.map((app) => ({ ...app, categories: app.categories.join("|") })), csvColumns)); writeText(path.join(outDir, "data", "apps.json"), JSON.stringify({ generated: date, source: `${site}/self-host-apps`, apps: listed }, null, 1)); - console.log(JSON.stringify({ out: outDir, apps: listed.length, repeatedRepositoryEntries: categorized.length - listed.length, withoutCategory: apps.length - categorized.length, categories: categories.length })); + console.log(JSON.stringify({ out: outDir, apps: listed.length, repeatedRepositoryEntries: repeatedRepositoryEntries(listed), withoutCategory: apps.length - categorized.length, categories: categories.length })); } if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { diff --git a/scripts/stack127/repos/sync-awesome-list.mjs b/scripts/stack127/repos/sync-awesome-list.mjs index afb7926d..6283f8fc 100644 --- a/scripts/stack127/repos/sync-awesome-list.mjs +++ b/scripts/stack127/repos/sync-awesome-list.mjs @@ -69,7 +69,7 @@ export function syncGenerated(source, destination, next = inspectExport(source)) for (const name of old) if (!next.files.includes(`categories/${name}`)) rmSync(path.join(destination, 'categories', name)); } function redact(text) { - for (const value of [process.env.AWESOME_LIST_GITHUB_TOKEN, process.env.DATABASE_URL]) if (value) text = text.replaceAll(value, '[redacted]'); + for (const value of [process.env.PUBLIC_REPOS_GITHUB_TOKEN, process.env.DATABASE_URL]) if (value) text = text.replaceAll(value, '[redacted]'); return text; } export function publishExport(source, destination, { git, allowCountDrop = false } = {}) { @@ -88,7 +88,7 @@ export function publishExport(source, destination, { git, allowCountDrop = false return true; } export function runWeekly({ workRoot = '/srv/stack127-data/self-host-apps', allowCountDrop = process.env.AWESOME_LIST_ALLOW_COUNT_DROP === '1' } = {}) { - if (!process.env.AWESOME_LIST_GITHUB_TOKEN?.trim()) throw new Error('AWESOME_LIST_GITHUB_TOKEN is missing'); + if (!process.env.PUBLIC_REPOS_GITHUB_TOKEN?.trim()) throw new Error('PUBLIC_REPOS_GITHUB_TOKEN is missing'); safePath(workRoot); mkdirSync(workRoot, { recursive: true }); const work = mkdtempSync(path.join(workRoot, 'awesome-list-')); @@ -96,7 +96,7 @@ export function runWeekly({ workRoot = '/srv/stack127-data/self-host-apps', allo const destination = path.join(work, 'repo'); const askpass = path.join(work, 'askpass.sh'); // Only the variable name goes on disk. The token is never in args/config/remote. - writeFileSync(askpass, '#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" x-access-token ;; *) printf "%s\\n" "$AWESOME_LIST_GITHUB_TOKEN" ;; esac\n', { mode: 0o700 }); + writeFileSync(askpass, '#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" x-access-token ;; *) printf "%s\\n" "$PUBLIC_REPOS_GITHUB_TOKEN" ;; esac\n', { mode: 0o700 }); const env = { ...process.env, GIT_AUTHOR_NAME: 'stack127 list bot', GIT_AUTHOR_EMAIL: 'awesome-list-bot@users.noreply.github.com', GIT_COMMITTER_NAME: 'stack127 list bot', GIT_COMMITTER_EMAIL: 'awesome-list-bot@users.noreply.github.com', GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: askpass, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' }; const run = (command, args, cwd) => { const result = spawnSync(command, args, { cwd, env, encoding: 'utf8', timeout: 300000, maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); diff --git a/stack127_docs/brand-growth/status.md b/stack127_docs/brand-growth/status.md index 17dabd0c..3028f77f 100644 --- a/stack127_docs/brand-growth/status.md +++ b/stack127_docs/brand-growth/status.md @@ -28,6 +28,7 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h - 2026-11-13: all 54 published category pages (42 self-host, 12 CLI) rebuilt with the main page's cards, filters, search, sort, and pagination, plus a "how to choose" section below the cards. Released in pull request #120, commit `070a5c79`. - 2026-11-13: 98 duplicate directory entries retired in production (4,913 to 4,815 published self-hostable apps). The decisions are in `scripts/self-host-apps/batches/2026-11-13-published-duplicates-reviewed.json`. The public list was refreshed by hand: 4,803 apps in 42 categories. - 2026-11-13: owner did the "Request indexing" clicks for the 10 hub URLs. +- 2026-11-13: retiring the 98 apps broke live search for about 20 minutes: `/api/self-host-apps/search` returned 503 for any result that included a retired app, until the frontend refresh run finished. Fixed the same day: the API now leaves out an app whose details are gone and does not cache that response. The public list now shows every published app, not one per repository. ## Next @@ -35,8 +36,7 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h | --- | --- | --- | | Request indexing for the 10 largest self-host category pages | Owner | Open | | Submit the remaining 148 sitemap URLs to Bing, 100 a day | Agent | Open | -| Install the weekly list refresh on OVH (`self-host-awesome-list`, Monday 14:02 UTC) | Agent | Code is on `main`. Needs `AWESOME_LIST_GITHUB_TOKEN` in Infisical `stack127 /vps/self-host-apps`: a fine-grained token limited to that repository with Contents read and write. Then follow `scripts/self-host-apps/ovh/README.md`, including the supervisor reinstall | -| Make search survive a retired app | Agent, after the owner's go | Open. See the gap below | +| Install the weekly list refresh on OVH (`self-host-awesome-list`, Monday 14:02 UTC) | Agent | Code is on `main`. Needs `PUBLIC_REPOS_GITHUB_TOKEN` in Infisical `stack127 /vps/self-host-apps`: one shared fine-grained token for the public output repositories (select repositories, Contents read and write), never the main repository. Then follow `scripts/self-host-apps/ovh/README.md`, including the supervisor reinstall | | Pull the numbers and write the first data story | Agent | Open. Use 4,815 apps | | Post the story on r/selfhosted | Owner | After review | | Measure referring domains as a baseline | Agent | Open | @@ -44,9 +44,7 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h ## Known data gaps -- Retiring apps broke live search for about 20 minutes on 2026-11-13. `/api/self-host-apps/search` returned 503 for any result that included a retired app, because the cached app index still listed it while its details were gone. It recovered when the frontend refresh run finished. `lib/self-host-apps-search-data.ts` throws "Missing search detail"; it should drop such an app from the result. - 11 possible duplicate groups were left untouched as unsure, for example "Borg" and "borgserver", and "ONLYOFFICE" and "ONLYOFFICE Docs". They are marked `unsure` in the batch file. -- The public list still shows one app per source repository, so 12 real apps that share a repository with another app, for example "Joplin Server", are missing from it (4,803 against 4,815). Now that the directory is clean, the export can list every published app. - The "Has mobile app" filter matches 0 to 4 apps per category in development, and "Needs database" is empty for most apps. Both need data work. - Two CLI categories, `communication` and `games-fun`, are below 15 apps and unpublished. Their intros are placeholder text. - The main repository `stack127labs/stack127` is public on GitHub for build-minute reasons. The owner plans to make it private later. @@ -55,4 +53,3 @@ The 30 indexed pages are the home page, `/about`, `/privacy`, `/terms`, `/self-h ## Waiting on the owner - The GitHub token for the weekly list refresh. -- A yes to release the search fix. diff --git a/stack127_docs/stack127-repos.md b/stack127_docs/stack127-repos.md index d4f16dd2..abf9da66 100644 --- a/stack127_docs/stack127-repos.md +++ b/stack127_docs/stack127-repos.md @@ -102,7 +102,7 @@ npm run stack127:repo:tool:new -- --slug port-checker --name "Port Checker" --dr Dataset exports read verified production data, pick public columns, and sort rows the same way every time. They show a dry-run diff before any reviewed write or push. -The awesome-list export builds the files of `stack127labs/awesome-self-hosted-apps`: a README with a category index, one Markdown table per published category page, and `data/apps.csv` and `data/apps.json`. It lists each source repository once, so its app count is lower than the directory's. The OVH `self-host-awesome-list` job refreshes the public repository on Mondays at 14:02 UTC, with hourly catch-up. It exports into a fresh directory outside the code checkout, validates the files, refuses app-count drops over 10% and any category-count drop, then commits only changed generated files and pushes to `main`. Each run uses a fresh clone outside the code checkout, so interrupted commits and failed pushes cannot leave pending local history for the next run. Unrelated repository files stay intact. Setup, the repository-only token, and the manual count override are documented in [the OVH runner guide](../scripts/self-host-apps/ovh/README.md). Built, waiting for install. +The awesome-list export builds the files of `stack127labs/awesome-self-hosted-apps`: a README with a category index, one Markdown table per published category page, and `data/apps.csv` and `data/apps.json`. It lists every published app that has a published category page. The OVH `self-host-awesome-list` job refreshes the public repository on Mondays at 14:02 UTC, with hourly catch-up. It exports into a fresh directory outside the code checkout, validates the files, refuses app-count drops over 10% and any category-count drop, then commits only changed generated files and pushes to `main`. Each run uses a fresh clone outside the code checkout, so interrupted commits and failed pushes cannot leave pending local history for the next run. Unrelated repository files stay intact. Setup, the repository-only token, and the manual count override are documented in [the OVH runner guide](../scripts/self-host-apps/ovh/README.md). Built, waiting for install. Exports must use `.env.database`, an explicit production `--database-url` where supported, or the awesome-list export’s `--database-env` option with the OVH runner’s validated production `DATABASE_URL`. They never fall back to `.env.local`. diff --git a/tests/self-host-apps/ovh-awesome-list.test.mjs b/tests/self-host-apps/ovh-awesome-list.test.mjs index ac69f550..40582162 100644 --- a/tests/self-host-apps/ovh-awesome-list.test.mjs +++ b/tests/self-host-apps/ovh-awesome-list.test.mjs @@ -110,10 +110,10 @@ test('env preflight requires the dedicated token only for the list job', t => { writeFileSync(wrapper, '#!/bin/bash\nexit 0\n'); const validator = new URL('../../scripts/self-host-apps/ovh/validate-env.mjs', import.meta.url); const env = { ...process.env, DATABASE_URL: 'postgresql://stack127_app@127.0.0.1:5432/postgres?sslmode=disable' }; - delete env.AWESOME_LIST_GITHUB_TOKEN; + delete env.PUBLIC_REPOS_GITHUB_TOKEN; const options = { env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe', 'ignore', 'ignore', 'ignore', 'ignore', 'ignore', 'ignore'] }; - assert.throws(() => execFileSync(process.execPath, [validator.pathname, 'self-host-awesome-list', wrapper, root, 'test'], options), error => /AWESOME_LIST_GITHUB_TOKEN is missing/.test(error.stderr)); + assert.throws(() => execFileSync(process.execPath, [validator.pathname, 'self-host-awesome-list', wrapper, root, 'test'], options), error => /PUBLIC_REPOS_GITHUB_TOKEN is missing/.test(error.stderr)); assert.doesNotThrow(() => execFileSync(process.execPath, [validator.pathname, 'self-host-discovery-lists', wrapper, root, 'test'], options)); - env.AWESOME_LIST_GITHUB_TOKEN = 'fixture-only'; + env.PUBLIC_REPOS_GITHUB_TOKEN = 'fixture-only'; assert.doesNotThrow(() => execFileSync(process.execPath, [validator.pathname, 'self-host-awesome-list', wrapper, root, 'test'], options)); }); diff --git a/tests/self-host-apps/ovh-checkout-recovery.test.mjs b/tests/self-host-apps/ovh-checkout-recovery.test.mjs index 2d486e3a..c356768a 100644 --- a/tests/self-host-apps/ovh-checkout-recovery.test.mjs +++ b/tests/self-host-apps/ovh-checkout-recovery.test.mjs @@ -181,7 +181,7 @@ test('installed supervisor logs checkout wrapper syntax failures, then fetches a .replaceAll('/srv/stack127-self-host', f.checkout) .replaceAll('/srv/stack127-data/logs', logs)); // Public dummy connection syntax passes preflight; no process connects to it. - writeFileSync(envFile, 'DATABASE_URL=postgresql://stack127_app@127.0.0.1:5432/postgres?sslmode=disable\nGITHUB_TOKEN=fixture-only\nAWESOME_LIST_GITHUB_TOKEN=fixture-only\n', { mode: 0o600 }); + writeFileSync(envFile, 'DATABASE_URL=postgresql://stack127_app@127.0.0.1:5432/postgres?sslmode=disable\nGITHUB_TOKEN=fixture-only\nPUBLIC_REPOS_GITHUB_TOKEN=fixture-only\n', { mode: 0o600 }); mkdirSync(path.join(f.checkout, 'node_modules')); writeFileSync(path.join(f.root, 'dependencies.sha256'), `${createHash('sha256').update(lockfile).digest('hex')}\n`); const run = () => execFileSync('/bin/bash', [installed, job, '--due'], { encoding: 'utf8', timeout: 30_000, stdio: ['ignore', 'pipe', 'pipe'] }); @@ -240,7 +240,7 @@ function supervisorFixture(t, steps = 'echo "fixture-only-job $job"') { writeFileSync(path.join(f.origin, 'package-lock.json'), lockfile); git(f.origin, 'add', '.'); git(f.origin, 'commit', '-m', 'safe supervisor fixture'); - writeFileSync(`${f.root}.env`, 'DATABASE_URL=postgresql://stack127_app@127.0.0.1:5432/postgres?sslmode=disable\nGITHUB_TOKEN=fixture-only\nAWESOME_LIST_GITHUB_TOKEN=fixture-only\n', { mode: 0o600 }); + writeFileSync(`${f.root}.env`, 'DATABASE_URL=postgresql://stack127_app@127.0.0.1:5432/postgres?sslmode=disable\nGITHUB_TOKEN=fixture-only\nPUBLIC_REPOS_GITHUB_TOKEN=fixture-only\n', { mode: 0o600 }); mkdirSync(path.join(f.checkout, 'node_modules')); writeFileSync(path.join(f.root, 'dependencies.sha256'), `${createHash('sha256').update(lockfile).digest('hex')}\n`); const run = (job = 'self-host-catch-up', args = []) => execFileSync('/bin/bash', [installed, job, ...args], { diff --git a/tests/stack127/awesome-list-export.test.mjs b/tests/stack127/awesome-list-export.test.mjs index 986baa44..c052faaf 100644 --- a/tests/stack127/awesome-list-export.test.mjs +++ b/tests/stack127/awesome-list-export.test.mjs @@ -3,18 +3,16 @@ import test from "node:test"; import { mkdirSync, mkdtempSync, symlinkSync, writeFileSync, existsSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { cell, httpUrl, licenseText, normalizeApp, onePerRepository, prepareOutDir, renderCategory, repositoryKey } from "../../scripts/stack127/repos/export-awesome-list.mjs"; +import { cell, httpUrl, licenseText, normalizeApp, prepareOutDir, renderCategory, repeatedRepositoryEntries, repositoryKey } from "../../scripts/stack127/repos/export-awesome-list.mjs"; const app = (overrides) => normalizeApp({ slug: "a", name: "A", summary: "S", categories: ["wikis"], website_url: null, source_code_url: null, docs_url: null, publisher_url: null, license: null, source_model: "open_source", maintenance_status: "active", docker_support: "official", stars: null, last_commit_at: null, ram_estimate_min_mb: null, ram_estimate_max_mb: null, ...overrides }); -test("a repository appears once and the repackaged entry is dropped", () => { - const kept = onePerRepository([ - app({ slug: "binhex-jellyfin", name: "binhex-jellyfin", source_code_url: "https://github.com/jellyfin/jellyfin/" }), - app({ slug: "jellyfin", name: "Jellyfin", source_code_url: "https://github.com/Jellyfin/jellyfin" }), - app({ slug: "closed", name: "Closed", website_url: "https://closed.example" }), - app({ slug: "closed-2", name: "Closed 2", website_url: "https://closed.example" }), - ]); - assert.deepEqual(kept.map((item) => item.slug), ["jellyfin", "closed", "closed-2"]); +test("apps that share a repository are counted, not dropped", () => { + assert.equal(repeatedRepositoryEntries([ + app({ slug: "joplin-server", source_code_url: "https://github.com/laurent22/joplin/" }), + app({ slug: "joplin", source_code_url: "https://github.com/Laurent22/joplin" }), + app({ slug: "closed", website_url: "https://closed.example" }), + ]), 1); }); test("table cells cannot break the Markdown table or inject markup", () => { diff --git a/tests/stack127/self-host-apps-search.test.mjs b/tests/stack127/self-host-apps-search.test.mjs index 3bd1c5e3..e4c94bc7 100644 --- a/tests/stack127/self-host-apps-search.test.mjs +++ b/tests/stack127/self-host-apps-search.test.mjs @@ -117,12 +117,13 @@ test("the search API caches successful results and never caches bad input, unpub let published = true; let fail = false; let calls = 0; + let complete = true; const body = { ...select([app("one")], params("")), detailApps: [app("one")], facets: selfHostAppsFacets([app("one")]) }; const { GET } = await loadWithImports("../../app/api/self-host-apps/search/route.ts", { "next/server": nextServer, "@/lib/self-host-apps-search": searchModule, "@/lib/page-registry": { getPublishedPage: async () => published ? {} : null }, - "@/lib/self-host-apps-search-data": { getSelfHostAppsSearch: async () => { calls++; if (fail) throw new Error("database unavailable"); return body; } }, + "@/lib/self-host-apps-search-data": { getSelfHostAppsSearchForApi: async () => { calls++; if (fail) throw new Error("database unavailable"); return { response: body, complete }; } }, }); const request = query => new Request(`https://stack127.com/api/self-host-apps/search${query}`); const ok = await GET(request("?sort=name")); @@ -131,22 +132,28 @@ test("the search API caches successful results and never caches bad input, unpub assert.ok(ok.headers.get("Cache-Tag").includes("page:api:self-host-apps:search")); assert.ok(ok.headers.get("Cache-Tag").includes("self-host-catalog")); assert.deepEqual(await ok.json(), body); + complete = false; + const partial = await GET(request("?sort=name")); + assert.equal(partial.status, 200); + assert.equal(partial.headers.get("Cache-Control"), "no-store"); + assert.equal(partial.headers.get("Cache-Tag"), null); + complete = true; const invalid = await GET(request("?page=-1")); assert.equal(invalid.status, 400); assert.equal(invalid.headers.get("Cache-Control"), "no-store"); - assert.equal(calls, 1); + assert.equal(calls, 2); published = false; const hidden = await GET(request("")); assert.equal(hidden.status, 404); assert.equal(hidden.headers.get("Cache-Control"), "no-store"); - assert.equal(calls, 1); + assert.equal(calls, 2); published = true; fail = true; const unavailable = await GET(request("")); assert.equal(unavailable.status, 503); assert.equal(unavailable.headers.get("Cache-Control"), "no-store"); }); -test("search cards and details stay in the same order, and missing details fail instead of returning an incomplete result", async () => { +test("search cards and details stay in the same order; a missing detail fails for pages and is left out for the API", async () => { const searchModule = await import("../../lib/self-host-apps-search.ts"); let details = [app("z"), app("a")]; const loaders = { @@ -164,6 +171,14 @@ test("search cards and details stay in the same order, and missing details fail assert.equal((await data.loadSelfHostAppsSearch(params(""))).apps[0].slug, "fresh"); details = [app("a")]; await assert.rejects(data.getSelfHostAppsSearch(params("")), /Missing search detail/); + const partial = await data.getSelfHostAppsSearchForApi(params("sort=name")); + assert.equal(partial.complete, false); + assert.deepEqual(partial.response.apps.map(app => app.slug), ["a"]); + assert.deepEqual(Array.from(partial.response.detailApps, app => app.slug), ["a"]); + details = []; + await assert.rejects(data.getSelfHostAppsSearchForApi(params("")), /Missing search details for every app/); + details = [app("z"), app("a")]; + assert.equal((await data.getSelfHostAppsSearchForApi(params(""))).complete, true); }); test("origin verification checks search against the source and rejects a stale response before the edge purge", async () => {