diff --git a/.env.example b/.env.example index 4b6523ef6..589bc7272 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,12 @@ HTML2RSS_ACCESS_TOKEN= # Keep enabled to allow pasting any webpage URL to generate an RSS feed. AUTO_SOURCE_ENABLED=true +# OPTIONAL: Config refinement studio (off in production unless set to true). +# Exposes POST /api/v1/feeds/validate, /preview and /suggest_selectors. All three +# require HTML2RSS_ACCESS_TOKEN; preview and suggest_selectors also fetch the +# submitted URL live and need AUTO_SOURCE_ENABLED=true. +# STUDIO_ENABLED=false + # COMPANION SCRAPER: # URL of the companion Botasaurus scraper API service in docker-compose. BOTASAURUS_SCRAPER_URL=http://botasaurus:4010 diff --git a/.gitignore b/.gitignore index 44b0b160b..1ea980c19 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ frontend/.astro # Local agent artifacts /.agents/ +.agents/plan diff --git a/AGENTS.md b/AGENTS.md index 598182552..cf13b55dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,19 +58,50 @@ See [docs/design-system.md](docs/design-system.md) for visual rules. Public feed-directory metadata for embedded and local configs. -| Item | Detail | -| --- | --- | -| Endpoint | `GET /api/v1/configs` | -| Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | -| Disabled response | `404` with `{ "error": "catalog_disabled" }` | -| Embedded entries | `Html2rss::Configs::Catalog.entries` — do not re-walk YAML in the handler | -| Local entries | `Catalog::Merge` includes `feeds.yml` feeds only when `directory.title` is set | -| Starter feeds (UI) | Server `meta.starters` via `Catalog::Starters.pick`; frontend `selectStarterFeeds(entries, starters)` | -| Catalog find | `findCatalogEntries` → multi-hit list under create URL; demotes `empty`/`error`; links via `catalogFeedHref` | -| last_result | Required on every catalog entry (`ok`/`empty`/`error`/`unknown`); process-local `Feeds::LastResults` | -| catalog_version | `2` (clients fail-closed on other versions) | -| CORS | Route-scoped on `/api/v1/configs` only (`GET`, `OPTIONS`) | -| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` | -| Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | +| Item | Detail | +| ------------------ | ------------------------------------------------------------------------------------------------------------ | +| Endpoint | `GET /api/v1/configs` | +| Flag | `CONFIG_CATALOG_ENABLED` (default `true`; set `false` to disable) | +| Disabled response | `404` with `{ "error": "catalog_disabled" }` | +| Embedded entries | `Html2rss::Configs::Catalog.entries` — do not re-walk YAML in the handler | +| Local entries | `Catalog::Merge` includes `feeds.yml` feeds only when `directory.title` is set | +| Starter feeds (UI) | Server `meta.starters` via `Catalog::Starters.pick`; frontend `selectStarterFeeds(entries, starters)` | +| Catalog find | `findCatalogEntries` → multi-hit list under create URL; demotes `empty`/`error`; links via `catalogFeedHref` | +| last_result | Required on every catalog entry (`ok`/`empty`/`error`/`unknown`); process-local `Feeds::LastResults` | +| catalog_version | `2` (clients fail-closed on other versions) | +| CORS | Route-scoped on `/api/v1/configs` only (`GET`, `OPTIONS`) | +| Root metadata | `GET /api/v1/` exposes `instance.catalog: { enabled, url }` and `instance.studio: { enabled }` | +| Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | After handler or envelope changes: `make openapi` and `make ci-ready`. + +## Config studio API + +Token-holder refinement of the selectors subtree, then generate via create. + +| Item | Detail | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Endpoints | `POST /api/v1/feeds/validate`, `POST /api/v1/feeds/preview`, `POST /api/v1/feeds/suggest_selectors` | +| Create | `POST /api/v1/feeds` accepts optional `selectors` (same access token and flag as validate; URL-only create is unaffected by the flag) | +| Flag | `STUDIO_ENABLED` (development/test default `true`; otherwise `false`); advertised as `instance.studio.enabled` on `GET /api/v1` | +| Disabled response | `403` when the flag is off | +| Auth | Access token required on validate, preview, suggest, and create | +| Live fetch | Preview and suggest also need `AUTO_SOURCE_ENABLED` | +| Frontend | Successful results refine inline on `#/result/:token`. There is no `#/refine` route. | +| Contract SSOT | Request specs under `spec/html2rss/web/api/v1_spec.rb` and generated `public/openapi.yaml` | + +After handler or envelope changes: `make openapi` and `make ci-ready`. + +`SelectorsDocument` is `Openapi::JsonSchema` output from `Html2rss::Config::Schema.json_schema` at generate time (full gem selectors subtree). Runtime ownership is `Html2rss::Web::SelectorsDocument`. The SPA refine draft uses `StudioSelectors` / `StudioDraft` (items, enhance, title, url, published_at only). The published document is OpenAPI 3.1. Preview sample, selector candidate, and validation-issue stamps stay runtime-owned in `Openapi::Document`. + +## Generated OpenAPI SPA transport + +The frontend JSON API is the generated Hey API client, not hand-written `fetch`. + +| Item | Detail | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Regen | `make openapi-client` (`frontend/openapi-ts.config.ts`). Never hand-edit `frontend/src/api/generated/**` | +| Value imports | Only `frontend/src/api/http/*.ts` and `src/__tests__/apiHttp*.test.ts`. Type-only imports are allowed elsewhere | +| Layers | Generated owns wire. Adapters in `frontend/src/api/http/` own transport (`unwrap` → `{ ok, status, body }`). Parsers own domain types | +| baseUrl | `/api/v1` same-origin. Do not change public OpenAPI `servers` or `RSpec::OpenAPI.servers` | +| Preview | JSON Feed stays `fetch(json_public_url)`. Do not use `renderFeedByToken` | diff --git a/CONTEXT.md b/CONTEXT.md index 81c2ec430..e97de8fcb 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -28,8 +28,17 @@ On create, `findCatalogEntries` matches the typed query against the catalog (URL ### CreateEntry remount Visiting create (including hashbang `#!/…` → `#/…`) bumps `createEntryKey` so the create surface remounts. Remount alone does not auto-submit; auto-submit requires `prefillUrl`. +### Config Studio +Selector refinement for a token holder when studio is enabled. One result frame has two phases: **unresolved** (create returned `EXTRACTION_EMPTY`, studio on, access token present — in-memory `#/result` with no feed token; the flat editor mounts immediately; **Save and generate feed** is primary; wire `Decision#message` is the notice) and **ready** (`#/result/:token` after items mint — **Copy feed URL** primary; flat editor mounts with zero clicks to reveal; **Save and generate feed** stays demoted until the draft changes; one meadow; live studio output replaces the result preview after an edit). Studio off or no token keeps empty extraction on create. The listing URL stays the subject; **Create another feed** starts a new listing. The visual builder and YAML are one document: validate exports `channel.url` plus selectors. A successful validate echo keeps the server’s items selector and enhance flag; the client keeps the field selectors it authored. Only `studioService` unwraps the studio envelope and maps ranked `candidates` (items / title / link / published) into domain suggestion buckets. Candidates never auto-apply. Suggestion outcomes are explicit: loading, ready, valid-empty, and failed+retry stay non-blocking while the editor remains editable. Optional Title / Link / Published fields appear only for selector, candidate, issue, or manual evidence; one **Add …** activator reveals dormant fields. Chips add or remove selector parts on the authored comma-joined string; suggest does not write that string, and chips do not change Enhance. Field extractors remain fixed by the draft mapper. Save posts studio selectors through Creation IO and stays on the result view (unresolved → ready when items mint). On ready, demoted **Open Feed Directory issue** opens a prefilled html2rss-configs GitHub issue (no tokens); it is a handoff for Riley, not SPA directory authoring. There is no `#/refine` route and no collapsed Refine disclosure. + +### SelectorsDocument +The full gem `selectors` subtree a token holder may attach on feed creation after the root-key gate (`CLIENT_ROOT_KEYS`) and gem schema validate. Runtime type `Html2rss::Web::SelectorsDocument`; OpenAPI component `SelectorsDocument`. Signed into the feed token and resolved on later serves. Not a full config: channel, headers, and strategy stay denied by the root-key gate (not a second web shape allowlist). + +### StudioSelectors +SPA studio outbound / draft projection only: `items` (selector + optional enhance), `title`, `url`, `published_at`. Hydrate fails closed when the document carries keys outside that subset (e.g. `order`, `pagination`, `guid`). Not the full `SelectorsDocument`. + ### Unmatched result -A result route whose in-memory created feed token does not match the route `feedToken`. Feed Flow replaces to create without prefill (no second journey decide elsewhere). +A result route without matching in-memory workspace: ready requires a created feed whose token matches (or leads) the route `feedToken`; unresolved requires in-memory empty-extraction workspace on bare `#/result`. Feed Flow replaces to create without prefill (no second journey decide elsewhere). ### Renderer Backend feed HTTP assembly: `Feeds::Renderer` owns the HTTP envelope and success serialization, and orchestrates `Feeds::FormatNegotiation` (Accept/path negotiation; `FormatNegotiation::MediaRange` for Accept scoring). Empty FEED bodies dump `ErrorClassifier::Decision#message`. diff --git a/Gemfile b/Gemfile index 6c1c1a75d..9796ea8cb 100644 --- a/Gemfile +++ b/Gemfile @@ -4,8 +4,7 @@ source 'https://rubygems.org' git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } -gem 'html2rss', '~> 0.30' -# gem 'html2rss', github: 'html2rss/html2rss', branch: 'master' +gem 'html2rss', github: 'html2rss/html2rss', branch: :master gem 'html2rss-configs', github: 'html2rss/html2rss-configs' # Use these instead of the two above (uncomment them) when developing locally: diff --git a/Gemfile.lock b/Gemfile.lock index 815ad48e4..3747ec463 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,27 @@ +GIT + remote: https://github.com/html2rss/html2rss + revision: fa2629e7d38bae48303d5642a5d021ada28a1e63 + branch: master + specs: + html2rss (0.30.0) + addressable (~> 2.7) + brotli + dry-validation + httpx (~> 1.8) + kramdown + mcp (~> 1.2) + mime-types (> 3.0) + nokogiri (>= 1.10, < 2.0) + rack (~> 3.0) + rackup (~> 2.0) + regexp_parser + rss + sanitize + thor + tzinfo + webrick (~> 1.9) + zeitwerk + GIT remote: https://github.com/html2rss/html2rss-configs revision: 2355ce1b4189b01282acfb99f8e4548fe3ec417a @@ -8,9 +32,9 @@ GIT GEM remote: https://rubygems.org/ specs: - actionpack (8.1.3.1) - actionview (= 8.1.3.1) - activesupport (= 8.1.3.1) + actionpack (8.1.4) + actionview (= 8.1.4) + activesupport (= 8.1.4) nokogiri (>= 1.8.5) rack (>= 2.2.4) rack-session (>= 1.0.1) @@ -18,13 +42,13 @@ GEM rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actionview (8.1.3.1) - activesupport (= 8.1.3.1) + actionview (8.1.4) + activesupport (= 8.1.4) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activesupport (8.1.3.1) + activesupport (8.1.4) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.3.1) @@ -40,18 +64,18 @@ GEM addressable (2.9.0) public_suffix (>= 2.0.2, < 8.0) ast (2.4.3) - async (2.45.1) + async (2.46.0) console (~> 1.29) fiber-annotation io-event (~> 1.21) async-container (0.38.0) async (~> 2.44) - async-http (0.103.0) + async-http (0.105.0) async (>= 2.35.1) async-pool (~> 0.12) io-endpoint (~> 0.18) io-stream (~> 0.14) - protocol-http (~> 0.66) + protocol-http (~> 0.72) protocol-http1 (~> 0.41) protocol-http2 (~> 0.28) protocol-url (~> 0.2) @@ -69,13 +93,13 @@ GEM bigdecimal samovar (~> 2.1) base64 (0.3.0) - bigdecimal (4.1.2) + bigdecimal (4.1.3) brotli (0.8.0) builder (3.3.0) climate_control (1.2.0) concurrent-ruby (1.3.8) connection_pool (3.0.2) - console (1.37.0) + console (1.38.0) fiber-annotation fiber-local (~> 1.1) json @@ -141,33 +165,15 @@ GEM fiber-storage (1.0.1) hana (1.3.7) hashdiff (1.2.1) - html2rss (0.30.0) - addressable (~> 2.7) - brotli - dry-validation - httpx (~> 1.8) - kramdown - mcp (~> 1.2) - mime-types (> 3.0) - nokogiri (>= 1.10, < 2.0) - rack (~> 3.0) - rackup (~> 2.0) - regexp_parser - rss - sanitize - thor - tzinfo - webrick (~> 1.9) - zeitwerk - http-2 (1.2.2) - httpx (1.8.3) + http-2 (1.2.3) + httpx (1.8.4) http-2 (>= 1.2.0) i18n (1.15.2) concurrent-ruby (~> 1.0) - io-console (0.9.2) - io-endpoint (0.18.0) + io-console (0.9.4) + io-endpoint (0.19.0) openssl (>= 3.3.0) - io-event (1.21.1) + io-event (1.22.1) io-stream (0.14.0) openssl (>= 3.3) irb (1.18.0) @@ -175,7 +181,7 @@ GEM prism (>= 1.3.0) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.21.2) + json (3.0.2) json_schemer (2.5.0) bigdecimal hana (~> 1.3) @@ -191,12 +197,12 @@ GEM loofah (2.25.2) crass (~> 1.0.2) nokogiri (>= 1.12.0) - mcp (1.5.0) + mcp (1.6.1) json_schemer (>= 2.4) mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2026.0701) + mime-types-data (3.2026.0929) minitest (6.0.6) drb (~> 2.0) prism (~> 1.5) @@ -217,7 +223,7 @@ GEM nokogiri (1.19.4-x86_64-linux-musl) racc (~> 1.4) openssl (4.0.2) - parallel (2.1.0) + parallel (2.3.0) parser (3.3.12.0) ast (~> 2.4.1) racc @@ -226,17 +232,18 @@ GEM prettyprint (0.2.0) prism (1.9.0) protocol-hpack (1.5.1) - protocol-http (0.71.0) + protocol-http (0.72.0) protocol-http1 (0.41.0) protocol-http (~> 0.68) - protocol-http2 (0.28.0) + protocol-http2 (0.29.1) protocol-hpack (~> 1.4) protocol-http (~> 0.62) - protocol-rack (0.22.1) + protocol-rack (0.23.0) + console (~> 1.0) io-stream (>= 0.10) protocol-http (~> 0.58) rack (>= 1.0) - protocol-url (0.18.0) + protocol-url (0.19.0) public_suffix (7.0.5) racc (1.8.1) rack (3.2.7) @@ -260,12 +267,12 @@ GEM logger prism (>= 1.6.0) tsort - rdoc (8.0.0) + rdoc (8.1.0) erb prism (>= 1.6.0) rbs (>= 4.0.0) tsort - regexp_parser (2.12.0) + regexp_parser (2.13.1) reline (0.7.0) io-console (~> 0.5) rexml (3.4.4) @@ -283,14 +290,14 @@ GEM rspec-mocks (3.13.8) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-openapi (0.33.1) + rspec-openapi (0.34.0) actionpack (>= 5.2.0) rails-dom-testing rspec-core rspec-support (3.13.7) rss (0.3.3) rexml - rubocop (1.90.0) + rubocop (1.91.0) json (>= 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -315,9 +322,9 @@ GEM lint_roller (~> 1.1) regexp_parser (>= 2.0) rubocop (~> 1.86, >= 1.86.2) - rubocop-thread_safety (0.7.3) + rubocop-thread_safety (0.8.0) lint_roller (~> 1.1) - rubocop (~> 1.72, >= 1.72.1) + rubocop (~> 1.89) rubocop-ast (>= 1.44.0, < 2.0) ruby-lsp (0.26.11) language_server-protocol (~> 3.17.0) @@ -330,11 +337,11 @@ GEM crass (~> 1.0.2) nokogiri (>= 1.16.8) securerandom (0.4.1) - sentry-ruby (7.0.0) + sentry-ruby (7.1.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) logger - simplecov (1.2.0) + simplecov (1.3.2) simpleidn (0.3.0) stackprof (0.2.28) string-format (0.2.0) @@ -342,9 +349,9 @@ GEM tsort (0.2.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (3.2.0) - unicode-emoji (~> 4.1) - unicode-emoji (4.2.0) + unicode-display_width (3.3.0) + unicode-emoji (~> 4.3) + unicode-emoji (4.3.0) uri (1.1.1) useragent (0.16.11) vcr (6.4.0) @@ -370,7 +377,7 @@ DEPENDENCIES base64 climate_control falcon - html2rss (~> 0.30) + html2rss! html2rss-configs! irb rack-test @@ -393,27 +400,27 @@ DEPENDENCIES zeitwerk CHECKSUMS - actionpack (8.1.3.1) sha256=974cb7154548e81f470b1b0f247b99cb38e87825899dca58610596e2817723d0 - actionview (8.1.3.1) sha256=2da68b8414c47b43bfbed1ce69c5afe1c04f78c267aacb5660a4cab5ca12cfb6 - activesupport (8.1.3.1) sha256=85458765f25ea48b9019c46b6bb3fa5683197bf4280d9f06710a6e8d7a831376 + actionpack (8.1.4) sha256=245493cc51be195a7645e6fc184880fa7e5b32ff1a08e053ee31f58c76e30ea4 + actionview (8.1.4) sha256=eff1efd07c4d969abad288322b423bcb67fdbab5e324c91ca37352b80f226076 + activesupport (8.1.4) sha256=891dd3fe43c4fc8391934b237b1189bb1f661c932a84a840043db9d46a4e80df addressable (2.9.0) sha256=7fdf6ac3660f7f4e867a0838be3f6cf722ace541dd97767fa42bc6cfa980c7af ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 - async (2.45.1) sha256=95849f7432eca535146c69f3078dab3c7228d0075ed71f48733f3e585dd7c267 + async (2.46.0) sha256=34f984733dbf63a997376346451ec995e858cacc3105e86ccfeaf8877b4f490b async-container (0.38.0) sha256=71479d8551870d684e602f463d7b8368909c835a64e38b555e1821512ffffc88 - async-http (0.103.0) sha256=137e3d5d419f7d72d5fa33cf04f30232e79c57b690025a650b8f9e48ece8bc41 + async-http (0.105.0) sha256=55191969d5cf61be5fbc2e0aadd51d77085318ba192a5ff5e12e3aba0826906a async-http-cache (0.4.6) sha256=2038d1f093182f16b50b4db271c25085e3938da10bfcfc2904cadb0530fddfd6 async-pool (0.12.0) sha256=5580493dc024bd704dab2ed0bf1307100722e49cbb825c7965f8501194f58506 async-service (0.25.0) sha256=71d9d75c2898dddf6549150cecd57156453e5b61da69fa87f10a0c97becfb6d1 async-utilization (0.5.0) sha256=96c347634e09edcbcb7f70cf8c5a1ae058c64d88186341e3f924a44440f95981 bake (0.25.0) sha256=a47bdc6a26addc048827debc36fe27bb4d5d71ac2958ad910b5386e9baf49869 base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b - bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd + bigdecimal (4.1.3) sha256=61ebe1e5e559bdc3cc6f2c0ee7f427321fc838f59611c294356eb04d6e21cf66 brotli (0.8.0) sha256=0c5a42046b3b603fb109656881147fd76064c034b7d19c1b4fcc32a093a4d55d builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f climate_control (1.2.0) sha256=36b21896193fa8c8536fa1cd843a07cf8ddbd03aaba43665e26c53ec1bd70aa5 concurrent-ruby (1.3.8) sha256=b2f1be836e968ccc78ccfce277ea79c72a88633f22306782c16ff23fb415d1e1 connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a - console (1.37.0) sha256=8093b286c2595a063849c098594fee5155ecc7967d36f3aa8cbe7569c8f3efd7 + console (1.38.0) sha256=35b82f59130e95e9be6b832d9e539c8b3988fc192e4a8e9a86ab4922f44ad5de crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e crass (1.0.7) sha256=94868719948664c89ddcaf0a37c65048413dfcb1c869470a5f7a7ceb5390b295 diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 @@ -434,17 +441,17 @@ CHECKSUMS fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1 hana (1.3.7) sha256=5425db42d651fea08859811c29d20446f16af196308162894db208cac5ce9b0d hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 - html2rss (0.30.0) sha256=a0ec169cf4a41954b2ec327625b15e81d718061a59a0bd27670ca01f238a2852 + html2rss (0.30.0) html2rss-configs (0.2.0) - http-2 (1.2.2) sha256=81b5d45f50fd4cd5f8c5d09651184bec9401e3ef169c3eb3e5b003d5614a92b9 - httpx (1.8.3) sha256=cb88f2285c4ef17164d803a640dc393d28722cba7f282ad0e7f9f926cd02dc47 + http-2 (1.2.3) sha256=f4e3a7000e9a8ab97595f09eac44c6ec10a2aa438360758124df0ca5bcb280d4 + httpx (1.8.4) sha256=1782ecde81eb4e961ad8735ec74a6fec8e60efeaa4592c6b2f55d72e19aaeca8 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 - io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08 - io-endpoint (0.18.0) sha256=720d186bcd584e8ba490bfc91dab332cc3c5ffee848b5b03894555cb275151aa - io-event (1.21.1) sha256=b3088f89ad67edf9be9fc2f48a5fae0790e92862487551eeb337b49098eb99e6 + io-console (0.9.4) sha256=a403304f32928ee85e56d457b4612ca43f2703b561a3258844992bd24085fbb1 + io-endpoint (0.19.0) sha256=547aaefe9a78bc57fe25f9f7661b460395ebdfda506f70fdf4cc71d2ae5ca578 + io-event (1.22.1) sha256=0dd054cefe4a29f1b8286604f332c14f3208d75f54902961e91e94d45e5e1600 io-stream (0.14.0) sha256=07d9b13ea2066ccbbc761fff8d69ad5a31e47561d31e02cc271f3ed9c1a8f1cd irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 - json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a + json (3.0.2) sha256=8e6d7e7b11384c21230430cef90b71f14849a34a1f4452796670f7c981bd19df json_schemer (2.5.0) sha256=2f01fb4cce721a4e08dd068fc2030cffd0702a7f333f1ea2be6e8991f00ae396 kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa language_server-protocol (3.17.0.6) sha256=5ef2c0c138f8267e1bc631d3328347d354f96724b0af22f2c79516120443b7f0 @@ -452,9 +459,9 @@ CHECKSUMS localhost (1.8.0) sha256=df7ea825b4f64949c588c17efac86bc47ddc4460d723778abe933b71759b2701 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 loofah (2.25.2) sha256=2007f746959ac65552456e04b433e83deb22759ab38c838b4445c70e43425918 - mcp (1.5.0) sha256=6f97785fa6e069eb667aa3d50299851ea9aca904b4abe8306f156bb8fd9eaff0 + mcp (1.6.1) sha256=5646140425a85fe39d416c8bd7d612ada8e1e589ff3306d5c010e1ba8656308e mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 - mime-types-data (3.2026.0701) sha256=cd8811e1fb89d836499ba0582368a10ee74cef929ba956d1d5ddca045e6a730f + mime-types-data (3.2026.0929) sha256=3632d6a08f6ff832ef1181365b24cfe365b82a97c47ff76d5ae363d6d695aedc minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1 nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f nokogiri (1.19.4-aarch64-linux-musl) sha256=35c65b9ce72b3bb03207bdbe7067915019dc18c1b9b59139684bd6690fdd01af @@ -465,17 +472,17 @@ CHECKSUMS nokogiri (1.19.4-x86_64-linux-gnu) sha256=379fae440b28915e3f19d752ce2dcf8465ed2b2fbefd2a7ca0dd497bc981a06a nokogiri (1.19.4-x86_64-linux-musl) sha256=17dfb7c1fa194ae02fbf7c51a7afc8d278045ab3fdacfd86f91d02d7b274470b openssl (4.0.2) sha256=1037ad2868ae58df9ad917891c0c0f9815a1172f6846d4bcdd508e4c2ee747c2 - parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 + parallel (2.3.0) sha256=f75a3e904101ce6a1ccb6b8dc800cbafd42166d83330192438dcf29395167a6f parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 pp (0.6.4) sha256=dfcb0fce700c41456265922884f9fe195d7fbb0674a3578e6c0f69588e82b570 prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91 - protocol-http (0.71.0) sha256=38577bf3df0bf0c3f7ee7e21b43fcb4bd7624e36f697dbe9c3eea344f14bd9ba + protocol-http (0.72.0) sha256=b153474785979495b9f2df78f0fdc2e02edc7e92c1f7ea00425eed73bd02194c protocol-http1 (0.41.0) sha256=9863874a1228948a896c3e7ea16d13a680c8f3ed50706007c78b7033018b0ea3 - protocol-http2 (0.28.0) sha256=b7c6195dcfa80ec2484670e9fd8fc21024cc654a391aa0792433aff1188c1691 - protocol-rack (0.22.1) sha256=1185d245927ef9849a603700d6991ca353bc89724fbf98efa4a4333ed62a9fc3 - protocol-url (0.18.0) sha256=b16f55983094b46d18e04b709333b260e4ab5962a32696728e33a6c396150ed6 + protocol-http2 (0.29.1) sha256=3f4ee667b5730b7d05c70d3771bd5d5cdbb31beedaa9d05f2bdac68c9c984d19 + protocol-rack (0.23.0) sha256=63eec3341b20565b3dd155345fd1426ecbbed322cdc8a6accc993905388c03d7 + protocol-url (0.19.0) sha256=3f6b9f6aa742a259a0b3610fbbd9685dcd9506adf6734f428ff96259faabfb21 public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f rack (3.2.7) sha256=93e13e1c24f93556671d85d2d79fa228c3485815c50d7e2f265b5330c6528fb7 @@ -487,8 +494,8 @@ CHECKSUMS rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 rbs (4.2.0) sha256=51f7b886dcc05bc09e10b901daa6a81829f6adc03101d6ca9ea4aac6103e0674 - rdoc (8.0.0) sha256=03bf8c08a9639658855a0cfd77c0abca8325c227693f7f33f82957811348c469 - regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb + rdoc (8.1.0) sha256=befc37880a5cfd36dcb176c748b5fd5ee39dd3536a75e81824e4972ce230f60a + regexp_parser (2.13.1) sha256=5aedb6b7c35688f51e86eef17a15374dfd287c83f4ca644b2c5f3ec50a33b44b reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 roda (3.108.0) sha256=a209acc5c885c7c9cfe703ad985b343026f79f67949932dd739b83e5a129ffbd @@ -496,30 +503,30 @@ CHECKSUMS rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 - rspec-openapi (0.33.1) sha256=8a039893400b8d8fe5c606c2f83d907fc674e74c6c2fcccf0e2dfeff3f5d6fd8 + rspec-openapi (0.34.0) sha256=ca3bf0faaa2c264c1e6de05fe8e311dee903b28c9a186ca23387c87491969a61 rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c rss (0.3.3) sha256=37ef1ec4d691d67edbe92159079a4e89a0965e6a6b32246009ae3d734d12d1ab - rubocop (1.90.0) sha256=9eb4c065b5c5154e4ef554c547972f3905a9eb6b53e657e580b6796b54bf8242 + rubocop (1.91.0) sha256=9c82b7bf391c5d7e3798c5b9996e22a1fe3bd7468e351dfdeb96140c058296d0 rubocop-ast (1.50.0) sha256=b9ca88300da0803ee222ad20cdb30494c0a784eed06fdc35d254b06d662788db rubocop-performance (1.27.0) sha256=eeeb1374d062a368ee1c787b70eb0b0cc4b184cb1f8565f424760946146d61ce rubocop-rake (0.7.1) sha256=3797f2b6810c3e9df7376c26d5f44f3475eda59eb1adc38e6f62ecf027cbae4d rubocop-rspec (3.10.2) sha256=0b3e2ecc592cd10ecbf0095bb58d1e357905276e069643523cc19eb7495f65e2 - rubocop-thread_safety (0.7.3) sha256=067cdd52fbf5deffc18995437e45b5194236eaff4f71de3375a1f6052e48f431 + rubocop-thread_safety (0.8.0) sha256=71fe5e7894fb9acfbd4c2dcd394feabc35bd690f75b0bc846e1504d9e31f9034 ruby-lsp (0.26.11) sha256=4cc8f1587237ff8c9031680b5491df901a82559eeaa1ae96863ffe700b32583d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 samovar (2.5.1) sha256=8a9fc41eb8868084f0321eb41678c485cfbc7d282fb306c0be67c3284b1d2394 sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 - sentry-ruby (7.0.0) sha256=e9616ff521355a983fad404ca575d8bb1f1949a8cac42a7a893dd02156bcdb1a - simplecov (1.2.0) sha256=ea6acd05eece5a41990e2a5171c57d15700d329326c7666c85ee8c6a0dd0977e + sentry-ruby (7.1.0) sha256=bd6d2d022d879268526f62f0dccfb76aac99542bfce355e2b6a49cd1a81c0102 + simplecov (1.3.2) sha256=2a22ec1759f1b8db60881a2e038758956fc69ef93ed669be0bde0678de6d0e85 simpleidn (0.3.0) sha256=12ca730bed2f3db04d11e9bfd1bca3e11fb37f55b21eb2e9793fb5814bf54d03 stackprof (0.2.28) sha256=4ec2ace02f386012b40ca20ef80c030ad711831f59511da12e83b34efb0f9a04 string-format (0.2.0) sha256=bc981c14116b061f12134549f32fa2d61a17b5a35dd6fd36596c21722a789af6 thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b - unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 - unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + unicode-display_width (3.3.0) sha256=4b7aa66a4b11db50f6f7e98411215cd0dd10eaecb665d36eec5bcacb3fa0b613 + unicode-emoji (4.3.0) sha256=11c02fa73290378c066bb0562cd4c87d8e1b706fbbe0059ca12746a4244de8ce uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 useragent (0.16.11) sha256=700e6413ad4bb954bb63547fa098dddf7b0ebe75b40cc6f93b8d54255b173844 vcr (6.4.0) sha256=077ac92cc16efc5904eb90492a18153b5e6ca5398046d8a249a7c96a9ea24ae6 @@ -529,4 +536,4 @@ CHECKSUMS zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5 BUNDLED WITH - 4.0.9 + 4.0.3 diff --git a/Makefile b/Makefile index c4444eb77..157cd9f62 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ # frozen_string_literal: true -.PHONY: help test lint lint-js lint-ruby lintfix lintfix-js lintfix-ruby setup dev clean frontend-setup check-frontend quick-check ready ci-ready yard-verify-public-docs openapi openapi-verify openapi-client openapi-client-verify openapi-lint openapi-lint-redocly openapi-lint-spectral openai-lint-spectral test-frontend-e2e lint-css-primitives +.PHONY: help test lint lint-js lint-ruby lintfix lintfix-js lintfix-ruby setup dev up down clean frontend-setup check-frontend quick-check ready ci-ready yard-verify-public-docs openapi openapi-verify openapi-client openapi-client-verify openapi-lint openapi-lint-redocly openapi-lint-spectral openai-lint-spectral test-frontend-e2e lint-css-primitives + +# Host-side local app. Distinct project so this does not stop the published web stack. +DEV_COMPOSE = docker compose -p html2rss-dev -f docker-compose.botasaurus.yml RUBOCOP_FLAGS ?= --cache false @@ -26,6 +29,15 @@ dev: ## Start development server with live reload @echo "Starting html2rss-web development environment..." @bin/dev +up: ## Local web app (this checkout) plus published Botasaurus + $(DEV_COMPOSE) up -d + BOTASAURUS_SCRAPER_URL=http://127.0.0.1:4010 bin/dev + +down: ## Stop the local web app and published Botasaurus + -pkill -f "falcon.*html2rss-web" || true + -pkill -f "vite.*4001" || true + $(DEV_COMPOSE) down + dev-ruby: ## Start Ruby server only @bin/dev-ruby @@ -42,11 +54,11 @@ test-ruby: ## Run Ruby tests only test-frontend: ## Run frontend tests only @cd frontend && pnpm run test:ci -test-frontend-unit: ## Run frontend unit tests only +test-frontend-unit: ## Run frontend Node unit tests only @cd frontend && pnpm run test:unit -test-frontend-contract: ## Run frontend contract tests only - @cd frontend && pnpm run test:contract +test-frontend-integration: ## Run frontend jsdom integration tests only + @cd frontend && pnpm run test:integration test-frontend-e2e: ## Run frontend Playwright smoke tests @cd frontend && pnpm run test:e2e @@ -111,10 +123,11 @@ ready: ## Pre-commit gate (quick checks + RSpec) bundle exec rspec @echo "Pre-commit checks complete!" -ci-ready: ## CI parity gate (ready + OpenAPI verify + frontend e2e smoke) +ci-ready: ## CI parity gate (ready + OpenAPI verify/lint + frontend e2e smoke) @echo "Running CI parity checks..." $(MAKE) ready $(MAKE) openapi-verify + $(MAKE) openapi-lint $(MAKE) test-frontend-e2e @echo "CI parity checks complete!" diff --git a/app.rb b/app.rb index 818c36b7f..629abb889 100644 --- a/app.rb +++ b/app.rb @@ -89,7 +89,9 @@ def development? = self.class.development? plugin :not_allowed plugin :exception_page plugin :error_handler do |error| - next exception_page(error) if development? && !error.is_a?(HttpError) + # Owned Decision mappings (EXTRACTION_EMPTY, BLOCKED_SURFACE, …) stay JSON API surface in + # development. Only unclassified INTERNAL_SERVER_ERROR gets the exception page. + next exception_page(error) if development? && development_exception_page?(error) ErrorResponder.respond(request: request, response: response, error: error) end @@ -108,6 +110,15 @@ def development? = self.class.development? private + # @param error [Exception] + # @return [Boolean] + def development_exception_page?(error) + return false if error.is_a?(HttpError) + return false if error.is_a?(ErrorClassifier::DecidedError) + + ErrorClassifier.classify(error).code == ErrorClassifier::INTERNAL_SERVER_ERROR.code + end + def render_index_page(router) router.response['Content-Type'] = 'text/html' index_path = index_page_path diff --git a/app/web/api/v1/create_feed.rb b/app/web/api/v1/create_feed.rb index b3638f57a..bc3dbb3a7 100644 --- a/app/web/api/v1/create_feed.rb +++ b/app/web/api/v1/create_feed.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'json' require 'time' module Html2rss @@ -9,15 +8,32 @@ module Api module V1 ## # Creates stable feed records from authenticated API requests. - module CreateFeed # rubocop:disable Metrics/ModuleLength + module CreateFeed FEED_ATTRIBUTE_KEYS = %i[id name url feed_token public_url json_public_url created_at updated_at].freeze - MAX_BODY_BYTES = 64 * 1024 ABSOLUTE_URL_REGEXP = %r{\A[a-z][a-z0-9+\-.]*://}i HOSTNAME_INPUT_REGEXP = %r{ \A(localhost(?::\d+)?|(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?|(?:[a-z0-9-]+\.)+[a-z]{2,}(?::\d+)?) (?:[/?#].*)?\z }ix + # Gates an optional selectors object through {SelectorsDocument} before it is signed. + # Absent keeps legacy tokens valid and keeps create on the automatic path, which the + # studio flag does not gate. Present means the caller authored the selectors in the + # studio, so the same kill switch the studio POSTs enforce applies here. + # + # @param request [Rack::Request] + # @return [Html2rss::Web::SelectorsDocument, nil] + # @raise [Html2rss::Web::ForbiddenError] when selectors are sent while the studio is disabled + def self.selectors_for(request) + raw = request_params(request)['selectors'] + return nil if raw.nil? + + raise Html2rss::Web::ForbiddenError, 'Studio is disabled' unless Flags.studio_enabled? + + SelectorsDocument.from_client({ selectors: raw }) + end + private_class_method :selectors_for + class << self # Creates a feed and returns a normalized API success payload. # @@ -26,8 +42,7 @@ class << self def call(request) account = require_account(request) params = build_create_params(request, account) - feed_data = create_feed(params, account) - + feed_data = create_feed(params, account, request) emit_create(status: :success, details: { url: params.url }) Response.success(response: request.response, status: 201, data: { feed: feed_attributes(feed_data) }, meta: { created: true }) @@ -39,14 +54,11 @@ def call(request) private def require_account(request) - account = Auth.authenticate(request) - raise Html2rss::Web::UnauthorizedError, 'Authentication required' unless account - - account + Auth.authenticate(request) || raise(Html2rss::Web::UnauthorizedError, 'Authentication required') end def build_create_params(request, account) - enforce_body_limit!(request) + JsonBody.enforce_limit!(request) params = request_params(request) url = validated_url(params['url'], account) name = params['name'].to_s.strip @@ -54,37 +66,10 @@ def build_create_params(request, account) FeedMetadata::CreateParams.new(url:, name:) end - def enforce_body_limit!(request) - return unless request.content_length.to_i > MAX_BODY_BYTES - - raise Html2rss::Web::BadRequestError, 'Payload too large' - end - def request_params(request) return request.params unless json_request?(request) - request.GET.merge(parsed_json_body(request)) - end - - def parsed_json_body(request) - raw_body = read_limited_body(request) - return {} if raw_body.strip.empty? - - parsed = JSON.parse(raw_body) - raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' unless parsed.is_a?(Hash) - - parsed - rescue JSON::ParserError - raise Html2rss::Web::BadRequestError, 'Invalid JSON payload' - end - - def read_limited_body(request) - enforce_body_limit!(request) - - request.body.read(MAX_BODY_BYTES + 1).to_s.tap do |raw_body| - request.body.rewind - raise Html2rss::Web::BadRequestError, 'Payload too large' if raw_body.bytesize > MAX_BODY_BYTES - end + request.GET.merge(JsonBody.object(request)) end def json_request?(request) @@ -117,15 +102,15 @@ def normalized_input_url(raw_url) HOSTNAME_INPUT_REGEXP.match?(url) ? "https://#{url}" : url end - # Extracts via {Feeds::Service} before minting so create and serve share one expansion path. - # # @param params [Html2rss::Web::Api::V1::FeedMetadata::CreateParams] # @param account [Hash] + # @param request [Rack::Request] # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata] - def create_feed(params, account) + def create_feed(params, account, request) + selectors = selectors_for(request) raise Html2rss::Web::AutoSourceDisabledError unless Flags.auto_source_enabled? - feed_token = Auth.generate_feed_token(account[:username], params.url) + feed_token = Auth.generate_feed_token(account[:username], params.url, selectors:) raise Html2rss::Web::InternalServerError, 'Failed to create feed' unless feed_token result = Feeds::Service.call(resolved_source_for(feed_token)) diff --git a/app/web/api/v1/json_body.rb b/app/web/api/v1/json_body.rb new file mode 100644 index 000000000..998057960 --- /dev/null +++ b/app/web/api/v1/json_body.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'json' + +module Html2rss + module Web + module Api + module V1 + ## + # Size-bounded JSON object reader for create and studio requests. + module JsonBody + MAX_BODY_BYTES = 64 * 1024 + + class << self + ## + # Parses the request body as a JSON object. + # + # An empty body is an empty object. A body over {MAX_BODY_BYTES} is rejected + # before it is parsed. The body is rewound so a later reader can see it. + # + # @param request [Rack::Request] + # @return [Hash] + # @raise [Html2rss::Web::BadRequestError] when the body is too large or not a JSON object + def object(request) + raw = read(request) + return {} if raw.strip.empty? + + parsed = JSON.parse(raw) + raise BadRequestError, 'Invalid JSON payload' unless parsed.is_a?(Hash) + + parsed + rescue JSON::ParserError + raise BadRequestError, 'Invalid JSON payload' + end + + ## + # Rejects a declared content length over {MAX_BODY_BYTES} without reading the body. + # + # @param request [Rack::Request] + # @return [void] + # @raise [Html2rss::Web::BadRequestError] when Content-Length is over the cap + def enforce_limit!(request) + return unless request.content_length.to_i > MAX_BODY_BYTES + + raise BadRequestError, 'Payload too large' + end + + private + + # @param request [Rack::Request] + # @return [String] + def read(request) + enforce_limit!(request) + + request.body.read(MAX_BODY_BYTES + 1).to_s.tap do |raw| + request.body.rewind + raise BadRequestError, 'Payload too large' if raw.bytesize > MAX_BODY_BYTES + end + end + end + end + end + end + end +end diff --git a/app/web/api/v1/preview_feed.rb b/app/web/api/v1/preview_feed.rb new file mode 100644 index 000000000..1b7bb50ed --- /dev/null +++ b/app/web/api/v1/preview_feed.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Api + module V1 + ## + # Live extraction preview for a token config carrying a selectors document. + # + # Builds the config through {Feeds::GeneratorInput} and selects + # {Html2rss::Test::Result} fields. Never forwards +#to_h+, which includes +rss:+. + # A later preview of the same canonical URL is replayed by {PreviewPageCache}. + module PreviewFeed + class << self + ## + # @param request [Rack::Request] + # @return [Hash{Symbol=>Object}] + def call(request) + account = StudioRequest.ensure_allowed!(request, live_fetch: true) + body = StudioRequest.json_object(request) + url = StudioRequest.allowed_url(body['url'], account) + config = PreviewPageCache.replay(url, preview_config(url, body['selectors'])) + result = Html2rss.test(config, min_items: 1) + PreviewPageCache.store(url, result) + Response.success(response: request.response, data: preview_fields(result)) + end + + private + + # @param url [String] + # @param selectors [Object] + # @return [Hash{Symbol=>Object}] + def preview_config(url, selectors) + Feeds::GeneratorInput.for_token(url:, selectors: selectors_document(selectors)) + end + + # @param selectors [Object] + # @return [Html2rss::Web::SelectorsDocument, nil] + def selectors_document(selectors) + return if selectors.nil? + + SelectorsDocument.from_client({ selectors: }) + end + + # Selected fields only. {Html2rss::Test::Result#to_h} also carries +rss+. + # Successful previews rebuild the meadow from that RSS, capped here. + # + # @param result [Html2rss::Test::Result] + # @return [Hash{Symbol=>Object}] + def preview_fields(result) + { + item_count: result.item_count, + sample_items: PreviewSamples.from_result(result), + quality_report: result.quality_report&.to_h, + failure_kind: result.failure_kind&.to_sym, + validation_issues: result.validation_issues&.map(&:to_h) + } + end + end + end + end + end + end +end diff --git a/app/web/api/v1/preview_page_cache.rb b/app/web/api/v1/preview_page_cache.rb new file mode 100644 index 000000000..6626e1f6c --- /dev/null +++ b/app/web/api/v1/preview_page_cache.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require 'tempfile' + +module Html2rss + module Web + module Api + module V1 + ## + # One process-local HTML page for studio preview replay. + # + # Keyed by the canonical URL {PreviewFeed} already allowed. A different + # URL replaces the slot after a successful fetch. Empty or failed bodies + # leave the stored page in place. The slot lasts for the process + # lifetime; process exit removes the temp file. + module PreviewPageCache + Slot = Data.define(:url, :path) + MUTEX = Mutex.new + + class << self + ## + # @param url [String] canonical page URL + # @return [String, nil] temp file path when this URL is cached + def path_for(url) + MUTEX.synchronize do + current = @slot + current.path if current&.url == url && File.file?(current.path) + end + end + + ## + # @param url [String] canonical page URL + # @param config [Hash{Symbol=>Object}] token config from {Feeds::GeneratorInput.for_token} + # @return [Hash{Symbol=>Object}] +config+, overlaid with +:local_file+ on a hit + def replay(url, config) + path = path_for(url) + return config unless path + + overlay(config, path) + end + + ## + # @param url [String] canonical page URL + # @param result [Html2rss::Test::Result] + # @return [void] + def store(url, result) + body = storable_body(result) + return unless body + + publish(url, write_body(body)) + end + + ## + # Drops the slot and its temp file. + # + # @return [void] + def clear! + publish_slot(nil) + end + + at_exit { Html2rss::Web::Api::V1::PreviewPageCache.clear! } + + private + + # @param config [Hash{Symbol=>Object}] + # @param path [String] + # @return [Hash{Symbol=>Object}] + def overlay(config, path) + request = config[:request].is_a?(Hash) ? config[:request].dup : {} + config.merge(strategy: :local_file, request: request.merge(local_file_path: path)) + end + + # A failed extraction and a blank page are not a new cached document. + # + # @param result [Html2rss::Test::Result] + # @return [String, nil] + def storable_body(result) + return unless result.success + + body = result.response_body + return unless body.is_a?(String) + return if body.strip.empty? + + body + end + + # @param url [String] + # @param path [String] + # @return [void] + def publish(url, path) + publish_slot(Slot.new(url:, path:)) + end + + # @param slot [Slot, nil] + # @return [void] + def publish_slot(slot) + previous = nil + MUTEX.synchronize do + previous = @slot + @slot = slot + end + unlink(previous.path) if previous && previous.path != slot&.path + end + + # @param body [String] + # @return [String] + def write_body(body) + file = Tempfile.create(['preview-page-', '.html']) + file.write(body) + file.path + ensure + file&.close + end + + # @param path [String, nil] + # @return [void] + def unlink(path) + File.unlink(path) if path + rescue Errno::ENOENT + nil + end + end + + private_constant :Slot, :MUTEX + end + end + end + end +end diff --git a/app/web/api/v1/preview_samples.rb b/app/web/api/v1/preview_samples.rb new file mode 100644 index 000000000..e3fbee6ee --- /dev/null +++ b/app/web/api/v1/preview_samples.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'nokogiri' +require 'rss' + +module Html2rss + module Web + module Api + module V1 + ## + # Maps a preview extraction into meadow rows. + # + # Gem samples stay at three items. The authenticated meadow cap is {SAMPLE_LIMIT} + # (the same 10 the member JSON Feed preview uses in + # frontend/src/feeds/feedParsers.ts as PREVIEW_ITEM_LIMIT.member). + # Rows use {SAMPLE_KEYS}. An unusable RSS document keeps the gem samples. + module PreviewSamples + SAMPLE_LIMIT = 10 + SAMPLE_KEYS = %w[title url published_at image].freeze + + class << self + ## + # RSS items when the document parsed; otherwise the gem samples. + # + # @param result [Html2rss::Test::Result] + # @return [Array] + def from_result(result) + from_rss = samples_from_rss(result.rss) + return from_rss unless from_rss.nil? + + result.sample_items + end + + private + + # @param rss [String, nil] + # @return [Array, nil] nil asks the caller to keep gem samples + def samples_from_rss(rss) + return unless rss.is_a?(String) && !rss.strip.empty? + + items = RSS::Parser.parse(rss, false)&.items.to_a + return if items.empty? + + items.first(SAMPLE_LIMIT).map { sample_from_item(it) } + rescue RSS::Error + nil + end + + # @param item [RSS::Rss::Channel::Item] + # @return [Hash] + def sample_from_item(item) + sample = { + 'title' => item.title.to_s.strip, + 'url' => item_link(item), + 'published_at' => published_at(item) + } + image = description_image_src(item) || image_enclosure_url(item) + image ? sample.merge('image' => image) : sample + end + + # @param item [RSS::Rss::Channel::Item] + # @return [String] + def item_link(item) + link = item.link if item.respond_to?(:link) + link = item.url if link.to_s.strip.empty? && item.respond_to?(:url) + link.to_s.strip + end + + # @param item [RSS::Rss::Channel::Item] + # @return [String, nil] + def published_at(item) + value = item.pubDate if item.respond_to?(:pubDate) + return if value.nil? + return value.utc.iso8601 if value.respond_to?(:utc) + + text = value.to_s.strip + text.empty? ? nil : text + end + + # First http(s) ++ in the item description. Article images are prepended there. + # + # @param item [RSS::Rss::Channel::Item] + # @return [String, nil] + def description_image_src(item) + html = item.description.to_s if item.respond_to?(:description) + return if html.nil? || html.empty? + + node = Nokogiri::HTML.fragment(html).css('img').find { http_image_url(it['src']) } + http_image_url(node['src']) if node + end + + # Image +enclosure+ only. Audio and other types are not preview images. + # + # @param item [RSS::Rss::Channel::Item] + # @return [String, nil] + def image_enclosure_url(item) + return unless item.respond_to?(:enclosure) + + enclosure = item.enclosure + type = enclosure.respond_to?(:type) ? enclosure.type.to_s : '' + return unless type.downcase.start_with?('image/') + + http_image_url(enclosure.url) + end + + # @param value [String, nil] + # @return [String, nil] + def http_image_url(value) + url = value.to_s.strip + return unless url.start_with?('https://', 'http://') + return if url.match?(/[[:space:]"'<>]/) + + url + end + end + end + end + end + end +end diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 562f5a86b..d04f6c1b7 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -27,14 +27,31 @@ def build(router) # @return [Hash{Symbol=>Object}] def instance_payload(router) { - feed_creation: { - enabled: Flags.auto_source_enabled?, - access_token_required: Flags.auto_source_enabled? - }, - catalog: { - enabled: Flags.config_catalog_enabled?, - url: "#{router.base_url}/api/v1/configs" - } + feed_creation: feed_creation_payload, + studio: studio_payload, + catalog: catalog_payload(router) + } + end + + # @return [Hash{Symbol=>Boolean}] + def feed_creation_payload + { + enabled: Flags.auto_source_enabled?, + access_token_required: Flags.auto_source_enabled? + } + end + + # @return [Hash{Symbol=>Boolean}] + def studio_payload + { enabled: Flags.studio_enabled? } + end + + # @param router [Roda::RodaRequest] + # @return [Hash{Symbol=>Object}] + def catalog_payload(router) + { + enabled: Flags.config_catalog_enabled?, + url: "#{router.base_url}/api/v1/configs" } end end diff --git a/app/web/api/v1/studio_request.rb b/app/web/api/v1/studio_request.rb new file mode 100644 index 000000000..9ffa5b753 --- /dev/null +++ b/app/web/api/v1/studio_request.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Api + module V1 + ## + # Shared authentication, feature gating, and JSON parsing for studio requests. + module StudioRequest + class << self + ## + # Authenticates before checking feature flags so anonymous callers + # cannot discover disabled studio capabilities. + # + # @param request [Rack::Request] + # @param live_fetch [Boolean] whether the request needs automatic source fetching + # @return [Hash] the authenticated account + def ensure_allowed!(request, live_fetch: false) + account = Auth.authenticate(request) + raise UnauthorizedError, 'Authentication required' unless account + raise ForbiddenError, 'Studio is disabled' unless Flags.studio_enabled? + raise AutoSourceDisabledError if live_fetch && !Flags.auto_source_enabled? + + account + end + + ## + # Canonical page URL the account is allowed to reach. + # + # @param raw_url [Object] + # @param account [Hash] + # @return [String] + def allowed_url(raw_url, account) + stripped = raw_url.to_s.strip + raise BadRequestError, 'URL parameter is required' if stripped.empty? + + require_allowed_url(stripped, account) + end + + ## + # Canonical page URL when the caller sent one. + # + # A blank value is absent. A present value that cannot be used raises. + # + # @param raw_url [Object] + # @param account [Hash] + # @return [String, nil] + def page_url(raw_url, account) + stripped = raw_url.to_s.strip + return if stripped.empty? + + require_allowed_url(stripped, account) + end + + ## + # Parses a size-bounded JSON request body as an object. + # + # @param request [Rack::Request] + # @return [Hash] + # @see JsonBody.object + def json_object(request) + JsonBody.object(request) + end + + private + + # @param stripped [String] + # @param account [Hash] + # @return [String] + def require_allowed_url(stripped, account) + url = UrlValidator.canonical_url(stripped) + raise BadRequestError, 'Invalid URL format' unless url + raise ForbiddenError, 'URL not allowed for this account' unless UrlValidator.url_allowed?(account, url) + + url + end + end + end + end + end + end +end diff --git a/app/web/api/v1/suggest_selectors.rb b/app/web/api/v1/suggest_selectors.rb new file mode 100644 index 000000000..c60efc355 --- /dev/null +++ b/app/web/api/v1/suggest_selectors.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Api + module V1 + ## + # Ranked capture-derived selector candidates for the refinement studio. + # + # +Html2rss.capture+ publishes buckets on +CaptureResult#candidates+. + # Empty buckets are a successful response. + module SuggestSelectors + # Upper bound for the client-supplied items selector hint. + MAX_ITEMS_SELECTOR_LENGTH = 256 + BUCKET_NAMES = %i[items title link published].freeze + + class << self + ## + # @param request [Rack::Request] + # @return [Hash{Symbol=>Object}] + def call(request) + account = StudioRequest.ensure_allowed!(request, live_fetch: true) + body = StudioRequest.json_object(request) + url = StudioRequest.allowed_url(body['url'], account) + captured = Html2rss.capture( + url, + strategy: :auto, + items_selector: hint(body['items_selector']), + **replay_options(url) + ) + Response.success(response: request.response, data: suggestion(captured)) + end + + private + + # @param url [String] + # @return [Hash{Symbol=>String}] + def replay_options(url) + path = PreviewPageCache.path_for(url) + path ? { local_file_path: path } : {} + end + + # @param raw [Object] + # @return [String, nil] + # @raise [Html2rss::Web::BadRequestError] when the hint exceeds the cap + def hint(raw) + text = raw.to_s.strip + raise BadRequestError, 'items_selector is too long' if text.length > MAX_ITEMS_SELECTOR_LENGTH + + text.empty? ? nil : text + end + + # @param captured [Html2rss::Capture::CaptureResult] + # @return [Hash{Symbol=>Object}] + def suggestion(captured) + { + candidates: candidate_buckets(captured.candidates), + segment_strategy: captured.segment_strategy&.to_s, + admission_drops: captured.admission_drops + } + end + + # @param raw [Hash, nil] + # @return [Hash{Symbol=>Array}] + def candidate_buckets(raw) + buckets = raw.is_a?(Hash) ? raw : {} + BUCKET_NAMES.to_h do |name| + entries = bucket(buckets, name) + [name, name == :items ? item_entries(entries) : field_entries(entries)] + end + end + + # @param buckets [Hash] + # @param name [Symbol] + # @return [Array] + def bucket(buckets, name) + value = buckets.fetch(name) { buckets[name.to_s] } + value.is_a?(Array) ? value : [] + end + + # @param entries [Array] + # @return [Array] + def item_entries(entries) + entries.filter_map { item_entry(it) } + end + + # @param entry [Object] + # @return [Hash, nil] + def item_entry(entry) + selector = selector_text(entry) + sample = sample_text(entry) + return unless selector && sample + + { selector:, enhance: enhanced?(entry), sample: } + end + + # @param entries [Array] + # @return [Array] + def field_entries(entries) + entries.filter_map do |entry| + selector = selector_text(entry) + sample = sample_text(entry) + { selector:, sample: } if selector && sample + end + end + + # @param entry [Object] + # @return [String, nil] + def selector_text(entry) + return unless entry.is_a?(Hash) + + text = (entry[:selector] || entry['selector']).to_s.strip + text.empty? ? nil : text + end + + # @param entry [Hash] + # @return [String, nil] + def sample_text(entry) + return unless entry.is_a?(Hash) + + text = (entry[:sample] || entry['sample']).to_s.strip + text.empty? ? nil : text + end + + # @param entry [Hash] + # @return [Boolean] + def enhanced?(entry) + flag = entry.key?(:enhance) ? entry[:enhance] : entry['enhance'] + flag == true + end + end + end + end + end + end +end diff --git a/app/web/api/v1/validate_config.rb b/app/web/api/v1/validate_config.rb new file mode 100644 index 000000000..1f0139602 --- /dev/null +++ b/app/web/api/v1/validate_config.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Api + module V1 + ## + # Server-authoritative config check for the refinement studio. + # + # Parses YAML or a selectors object, gates it through {SelectorsDocument}, + # and returns the gem validation report plus normalized YAML. + module ValidateConfig + PLACEHOLDER_URL = 'https://example.com/' + private_constant :PLACEHOLDER_URL + + class << self + ## + # @param request [Rack::Request] + # @return [Hash{Symbol=>Object}] + def call(request) + account = StudioRequest.ensure_allowed!(request) + body = StudioRequest.json_object(request) + url = StudioRequest.page_url(body['url'], account) + report, selectors = assess(body, url) + Response.success(response: request.response, data: payload(report, selectors, url)) + end + + private + + # @param body [Hash] + # @param url [String, nil] + # @return [Array(Html2rss::Config::ValidationReport, Hash, nil)] + def assess(body, url) + reject_denied_body_keys!(body) + return assess_yaml(body['yaml'], url) if body.key?('yaml') && !body['yaml'].nil? + raise BadRequestError, 'yaml or selectors is required' unless body.key?('selectors') + + assess_fragment({ selectors: body['selectors'] }, url) + end + + # Transport keys (`url`, `selectors`, `yaml`) stay; denied config roots do not. + # + # @param body [Hash] + # @return [void] + def reject_denied_body_keys!(body) + denied = body.each_key.map { |key| key.to_s.to_sym }.find { SelectorsDocument::DENIED_ROOT_KEYS.include?(it) } + raise BadRequestError, "#{denied} is not allowed" if denied + end + + # @param yaml [Object] + # @param url [String, nil] + # @return [Array(Html2rss::Config::ValidationReport, Hash, nil)] + def assess_yaml(yaml, url) + assess_fragment(Html2rss::Config.from_yaml(yaml.to_s), url) + rescue ArgumentError, Psych::Exception => error + [Html2rss::Config::IssueMapper.parse_failure(error.message), nil] + end + + # A schema failure is a reportable outcome; anything else the root key + # gate rejects (denied or unknown root key, oversized wire) stays a 400. + # + # @param fragment [Hash] + # @param url [String, nil] + # @return [Array(Html2rss::Config::ValidationReport, Hash, nil)] + def assess_fragment(fragment, url) + document = SelectorsDocument.from_client(fragment) + [validation_report(document.selectors, url), document.selectors] + rescue SelectorsDocument::SchemaInvalid => error + selectors = fragment.is_a?(Hash) ? fragment[:selectors] || fragment['selectors'] : nil + [error.report, selectors.is_a?(Hash) ? selectors : nil] + end + + # The gem validator needs a channel URL. The export uses the caller's + # page URL and never the placeholder. + # + # @param selectors [Hash, nil] + # @param url [String, nil] + # @return [Html2rss::Config::ValidationReport] + def validation_report(selectors, url) + Html2rss::Config.validate({ channel: { url: url || PLACEHOLDER_URL }, selectors: selectors || {} }) + end + + # @param report [Html2rss::Config::ValidationReport] + # @param selectors [Hash, nil] + # @param url [String, nil] + # @return [Hash{Symbol=>Object}] + def payload(report, selectors, url) + { report: report.to_h, selectors:, yaml: export_yaml(selectors, url) } + end + + # @param selectors [Hash, nil] + # @param url [String, nil] + # @return [String, nil] + def export_yaml(selectors, url) + return unless selectors.is_a?(Hash) + + document = url ? { channel: { url: }, selectors: } : { selectors: } + Html2rss::Config.to_yaml(document) + end + end + end + end + end + end +end diff --git a/app/web/config/flags.rb b/app/web/config/flags.rb index 33a343fbe..e633f56df 100644 --- a/app/web/config/flags.rb +++ b/app/web/config/flags.rb @@ -24,6 +24,13 @@ module Flags # rubocop:disable Metrics/ModuleLength default: -> { development_or_test? }, validator: nil ), + studio_enabled: Definition.new( + name: :studio_enabled, + env_key: 'STUDIO_ENABLED', + type: :boolean, + default: -> { development_or_test? }, + validator: nil + ), config_catalog_enabled: Definition.new( name: :config_catalog_enabled, env_key: 'CONFIG_CATALOG_ENABLED', @@ -68,7 +75,7 @@ module Flags # rubocop:disable Metrics/ModuleLength ) }.freeze MANAGED_ENV_PREFIXES = %w[ - AUTO_SOURCE_ CONFIG_CATALOG_ FEEDS_CACHE_ RATE_LIMIT_ RETRY_AFTER_ + AUTO_SOURCE_ CONFIG_CATALOG_ FEEDS_CACHE_ RATE_LIMIT_ RETRY_AFTER_ STUDIO_ ].freeze KNOWN_ENV_KEYS = Set.new(DEFINITIONS.values.map(&:env_key)).freeze DEV_OR_TEST_ENVS = Set['development', 'test'].freeze @@ -109,6 +116,11 @@ def auto_source_enabled? fetch(:auto_source_enabled) end + # @return [Boolean] + def studio_enabled? + fetch(:studio_enabled) + end + # Validates all known flags and managed env key prefixes. # # @return [void] @@ -191,9 +203,13 @@ def validate_unknown_feature_keys! raise ArgumentError, "Unknown feature flags: #{unknown.sort.join(', ')}" end + # An unset RACK_ENV is not development. {EnvironmentValidator} already treats + # it as production, so defaulting it to development here would open the studio + # and live-fetch surfaces on a deployment that merely forgot the variable. + # # @return [Boolean] def development_or_test? - DEV_OR_TEST_ENVS.include?(ENV.fetch('RACK_ENV', 'development')) + DEV_OR_TEST_ENVS.include?(ENV.fetch('RACK_ENV', nil)) end end end diff --git a/app/web/feeds/generator_input.rb b/app/web/feeds/generator_input.rb new file mode 100644 index 000000000..f315365f5 --- /dev/null +++ b/app/web/feeds/generator_input.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Html2rss + module Web + module Feeds + ## + # Single owner of token-feed config expansion. + # + # Serve and studio preview both call {.for_token} so a preview cannot + # diverge from the feed a reader later receives. + module GeneratorInput + SUPPORTED_STRATEGIES = Set.new(Html2rss::RequestService.strategy_names.map(&:to_s)).freeze + private_constant :SUPPORTED_STRATEGIES + + class << self + ## + # Builds the generator config for a signed token feed. + # + # A nil or blank +strategy+ resolves to the configured default, or +auto+ + # when that default is blank. An explicit name must be supported. + # + # A nil or empty +selectors+ document keeps +auto_source+ so the gem runs + # automatic extraction. A non-empty document replaces that path: the + # returned config has the signed selectors and no +auto_source+ key, so + # the gem does not also run the general scrapers. + # + # @param url [String] channel URL bound to the token + # @param strategy [String, Symbol, nil] token strategy, or nil for the configured default + # @param selectors [Html2rss::Web::SelectorsDocument, nil] signed selector fragment + # @return [Hash{Symbol=>Object}] + def for_token(url:, strategy: nil, selectors: nil) + expanded = LocalConfig.global.slice(:stylesheets, :headers).merge( + channel: { url: }, + strategy: resolve_strategy(strategy).to_sym + ) + return expanded.merge(auto_source: {}) if selectors.nil? || selectors.empty? + + expanded.merge(selectors.to_config_fragment) + end + + private + + # @param strategy [String, Symbol, nil] + # @return [String] + # @raise [Html2rss::Web::BadRequestError] when the name is not supported + def resolve_strategy(strategy) + name = strategy.to_s.strip + return default_strategy_name if name.empty? + return name if name == default_strategy_name + raise Html2rss::Web::BadRequestError, 'Unsupported strategy' unless SUPPORTED_STRATEGIES.include?(name) + + name + end + + # @return [String] + def default_strategy_name + configured = Html2rss::Config.default_strategy_name.to_s + return configured unless configured.strip.empty? + + 'auto' + end + end + end + end + end +end diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index c96d185c3..08aec4fa2 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -8,8 +8,6 @@ module Feeds ## # Resolves static and token-backed requests into shared generator inputs. module SourceResolver - SUPPORTED_STRATEGIES = Set.new(Html2rss::RequestService.strategy_names.map(&:to_s)).freeze - class << self # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] @@ -63,7 +61,7 @@ def resolve_token(feed_request) # @param feed_token [Html2rss::Web::FeedToken] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def build_token_source(feed_request, feed_token) - generator_input = token_generator_input(feed_token.url, resolved_strategy(feed_token)) + generator_input = expanded_generator_input(feed_token) resolved_source( source_kind: :token, cache_identity: token_cache_identity(feed_request.token), @@ -75,6 +73,20 @@ def build_token_source(feed_request, feed_token) ) end + # Re-runs the full allowlist, including schema validation, against the wire + # document. Reached only after {#authorize_feed_token!} verified the signature. + # + # @param feed_token [Html2rss::Web::FeedToken] + # @return [Hash{Symbol=>Object}] + def expanded_generator_input(feed_token) + selectors = feed_token.selectors + GeneratorInput.for_token( + url: feed_token.url, + strategy: feed_token.strategy, + selectors: selectors.nil? ? nil : SelectorsDocument.from_verified_wire(selectors.to_wire) + ) + end + # @param kwargs [Hash{Symbol=>Object}] # @return [Html2rss::Web::Feeds::Contracts::ResolvedSource] def resolved_source(**kwargs) @@ -127,36 +139,6 @@ def authorize_feed_token!(token) raise Html2rss::Web::ForbiddenError, 'Access Denied' end - - # @param feed_token [Html2rss::Web::FeedToken] - # @return [String] - def resolved_strategy(feed_token) - strategy = feed_token.strategy.to_s.strip - return default_strategy_name if strategy.empty? - return strategy if strategy == default_strategy_name - raise Html2rss::Web::BadRequestError, 'Unsupported strategy' unless SUPPORTED_STRATEGIES.include?(strategy) - - strategy - end - - # @param url [String] - # @param strategy [String] - # @return [Hash{Symbol=>Object}] - def token_generator_input(url, strategy) - LocalConfig.global.slice(:stylesheets, :headers).merge( - channel: { url: }, - auto_source: {}, - strategy: strategy.to_sym - ) - end - - # @return [String] - def default_strategy_name - configured = Html2rss::Config.default_strategy_name.to_s - return configured unless configured.strip.empty? - - 'auto' - end end end end diff --git a/app/web/request/rate_limiter.rb b/app/web/request/rate_limiter.rb index 28fb06a80..4a665fe6a 100644 --- a/app/web/request/rate_limiter.rb +++ b/app/web/request/rate_limiter.rb @@ -10,6 +10,15 @@ module Web # Rack middleware providing IP-based rate limiting with fiber-native tracking, # automated pruning, and standardized 429 error formatting. class RateLimiter + # Live-fetch weights. Create is +POST /api/v1/feeds+ because that route runs + # {Feeds::Service}. Studio paths that only validate stay at one slot. + LIVE_FETCH_PATH_COSTS = { + 'POST /api/v1/feeds' => 5, + 'POST /api/v1/feeds/preview' => 5, + 'POST /api/v1/feeds/suggest_selectors' => 5 + }.freeze + private_constant :LIVE_FETCH_PATH_COSTS + ## # Encapsulates timestamp tracking and rate limit logic for a single client IP. class RequestTrack @@ -23,11 +32,15 @@ def initialize # Records request time, prunes old timestamps, and checks if limit is exceeded. # + # Admission looks at the current size, then appends +cost+ timestamps. + # A budget of N can overrun by cost - 1. + # # @param now [Integer] # @param window_seconds [Integer] # @param max_requests [Integer] + # @param cost [Integer] slots consumed by this request # @return [Array<(Boolean, Integer)>] limit exceeded flag and retry_after seconds. - def record_and_check_limit(now, window_seconds, max_requests) + def record_and_check_limit(now, window_seconds, max_requests, cost: 1) window_start = now - window_seconds @timestamps.reject! { |t| t < window_start } @@ -36,7 +49,7 @@ def record_and_check_limit(now, window_seconds, max_requests) retry_after = [1, oldest + window_seconds - now].max [true, retry_after] else - @timestamps << now + @timestamps.concat([now] * cost) [false, 0] end end @@ -80,16 +93,16 @@ def call(env) limit_exceeded, retry_after = track.record_and_check_limit( now, Flags.rate_limit_window_seconds, - Flags.rate_limit_max_requests + Flags.rate_limit_max_requests, + cost: request_cost(request, path) ) if limit_exceeded SecurityLogger.log_rate_limit_exceeded(client_key, path, Flags.rate_limit_max_requests) - # Ensure feed endpoints get feed-formatted errors - if path.start_with?('/api/v1/feeds/') || !path.start_with?('/api/v1/') - env[RequestTarget::ENV_KEY] = RequestTarget::FEED - end + # Ensure token and public feed endpoints get feed-formatted errors. + # Studio POSTs remain API-shaped even though they share the feeds prefix. + env[RequestTarget::ENV_KEY] = RequestTarget::FEED if feed_response_path?(path) error = TooManyRequestsError.new response = Rack::Response.new @@ -106,6 +119,31 @@ def call(env) private + # @param request [Rack::Request] + # @param path [String] + # @return [Integer] + def request_cost(request, path) + LIVE_FETCH_PATH_COSTS.fetch("#{request.request_method} #{path}", 1) + end + + # Studio POST names come from {Routes::ApiV1::FeedRoutes::STUDIO_POSTS}. + # + # @param path [String] + # @return [Boolean] + def studio_path?(path) + Routes::ApiV1::FeedRoutes::STUDIO_POSTS.each_key.any? do |name| + path == "/api/v1/feeds/#{name}" + end + end + + # @param path [String] + # @return [Boolean] + def feed_response_path?(path) + return true unless path.start_with?('/api/v1/') + + path.start_with?('/api/v1/feeds/') && !studio_path?(path) + end + # Bypasses rate limiting for root, static assets, and health checks. # # @param path [String] diff --git a/app/web/request/request_context_middleware.rb b/app/web/request/request_context_middleware.rb index 745eecb8e..3f2819c16 100644 --- a/app/web/request/request_context_middleware.rb +++ b/app/web/request/request_context_middleware.rb @@ -51,6 +51,9 @@ def route_group_for(path) 'static' end + # Reads the query string only. Touching +params+ would make Rack buffer the + # whole request body here, ahead of rate limiting and every handler body cap. + # # @param request [Rack::Request] # @return [Html2rss::Web::RequestContext::Context] def build_context(request) @@ -61,7 +64,7 @@ def build_context(request) http_method: request.request_method.to_s.upcase, route_group: route_group_for(path), actor: nil, - strategy: request.params['strategy'], + strategy: request.GET['strategy'], started_at: Time.now.utc.iso8601 ) end diff --git a/app/web/routes/api_v1/feed_routes.rb b/app/web/routes/api_v1/feed_routes.rb index 2c54cce0f..bdf48727b 100644 --- a/app/web/routes/api_v1/feed_routes.rb +++ b/app/web/routes/api_v1/feed_routes.rb @@ -7,23 +7,44 @@ module ApiV1 ## # Mounts feed-related API routes under `/api/v1/feeds`. module FeedRoutes + STUDIO_POSTS = { + 'validate' => Api::V1::ValidateConfig, + 'preview' => Api::V1::PreviewFeed, + 'suggest_selectors' => Api::V1::SuggestSelectors + }.freeze + class << self # @param router [Roda::RodaRequest] # @return [void] def call(router) router.on 'feeds' do - router.get String do |token| - router.env[RequestTarget::ENV_KEY] = RequestTarget::FEED - Feeds::Responder.call(request: router, target_kind: :token, identifier: token) - end + mount_studio_posts(router) + mount_token_get(router) - router.post do - JSON.generate(Api::V1::CreateFeed.call(router)) - end + router.post { JSON.generate(Api::V1::CreateFeed.call(router)) } raise NotFoundError end end + + private + + # @param router [Roda::RodaRequest] + # @return [void] + def mount_studio_posts(router) + STUDIO_POSTS.each do |path, handler| + router.post(path) { JSON.generate(handler.call(router)) } + end + end + + # @param router [Roda::RodaRequest] + # @return [void] + def mount_token_get(router) + router.get(String) do |token| + router.env[RequestTarget::ENV_KEY] = RequestTarget::FEED + Feeds::Responder.call(request: router, target_kind: :token, identifier: token) + end + end end end end diff --git a/app/web/security/auth.rb b/app/web/security/auth.rb index 89e5cca62..11fd27c8e 100644 --- a/app/web/security/auth.rb +++ b/app/web/security/auth.rb @@ -25,13 +25,15 @@ def authenticate(request) # @param username [String] # @param url [String] # @param strategy [String, nil] + # @param selectors [Html2rss::Web::SelectorsDocument, nil] # @param expires_in [Integer] seconds (default: 10 years) # @return [String, nil] signed feed token when generation succeeds. - def generate_feed_token(username, url, strategy: nil, expires_in: FeedToken::DEFAULT_EXPIRY) + def generate_feed_token(username, url, strategy: nil, selectors: nil, expires_in: FeedToken::DEFAULT_EXPIRY) token = FeedToken::Signer.create( username: username, url: url, strategy: strategy, + selectors: selectors, expires_in: expires_in, secret_key: secret_key ) diff --git a/app/web/security/feed_token.rb b/app/web/security/feed_token.rb index 37505d1ee..e2c0c0119 100644 --- a/app/web/security/feed_token.rb +++ b/app/web/security/feed_token.rb @@ -12,7 +12,20 @@ module Web # # Wire encoding and signature verification are unified inside this class to # keep the token lifetime concerns cohesive. - FeedToken = Data.define(:username, :url, :expires_at, :signature, :strategy) do + FeedToken = Data.define(:username, :url, :expires_at, :signature, :strategy, :selectors) do + ## + # @param username [String] + # @param url [String] + # @param expires_at [Integer] + # @param signature [String] + # @param strategy [String, nil] + # @param selectors [Html2rss::Web::SelectorsDocument, nil] + # @return [Html2rss::Web::FeedToken] + # rubocop:disable-next Metrics/ParameterLists + def initialize(username:, url:, expires_at:, signature:, strategy:, selectors: nil) + super + end + # @return [Boolean] def expired? Time.now.to_i > expires_at @@ -27,6 +40,12 @@ def valid_for_url?(candidate_url) class FeedToken DEFAULT_EXPIRY = 315_360_000 + # Largest encoded token accepted from the wire. A token carrying the maximum + # signed selectors subtree stays well under this. + MAX_ENCODED_BYTES = 4_096 + # Hard cap on the inflated wire document so an unauthenticated caller cannot + # trade a small token for a large allocation. + MAX_INFLATED_BYTES = 16_384 ## # zlib + URL-safe Base64 codec for feed-token wire payloads. @@ -43,12 +62,13 @@ def encode(token) # @return [Html2rss::Web::FeedToken, nil] def decode(encoded_token) return unless encoded_token + return if encoded_token.bytesize > MAX_ENCODED_BYTES token_data = parse_token_data(encoded_token) return unless valid_token_data?(token_data) decoded_token(token_data) - rescue JSON::ParserError, ArgumentError, Zlib::DataError, Zlib::BufError + rescue JSON::ParserError, ArgumentError, Zlib::DataError, Zlib::BufError, BadRequestError nil end @@ -59,16 +79,35 @@ def decode(encoded_token) def wire_document(token) payload = { u: token.username, l: token.url, e: token.expires_at } payload[:t] = token.strategy if token.strategy + payload[:c] = token.selectors.to_wire if signable_selectors?(token.selectors) { p: payload, s: token.signature } end # @param encoded_token [String] # @return [Hash{Symbol=>Object}] def parse_token_data(encoded_token) - inflated = Zlib::Inflate.inflate(Base64.urlsafe_decode64(encoded_token)) + inflated = inflate_bounded(Base64.urlsafe_decode64(encoded_token)) JSON.parse(inflated, symbolize_names: true) end + # Inflates in chunks and aborts once {MAX_INFLATED_BYTES} is exceeded, so a + # compression bomb is discarded instead of being fully expanded. + # + # @param compressed [String] + # @return [String] + # @raise [Zlib::DataError] when the document inflates past the cap + def inflate_bounded(compressed) + stream = Zlib::Inflate.new + buffer = +'' + stream.inflate(compressed) do |chunk| + buffer << chunk + raise Zlib::DataError, 'token payload exceeds the inflate cap' if buffer.bytesize > MAX_INFLATED_BYTES + end + buffer + ensure + stream.close + end + # @param token_data [Hash{Symbol=>Object}] # @return [Html2rss::Web::FeedToken] def decoded_token(token_data) @@ -78,10 +117,17 @@ def decoded_token(token_data) url: payload[:l], expires_at: payload[:e], signature: token_data[:s], - strategy: payload[:t] + strategy: payload[:t], + selectors: selectors_from_wire(payload[:c]) ) end + # @param wire [Object] + # @return [Boolean] + def valid_selectors_wire?(wire) + wire.nil? || wire.is_a?(Hash) + end + # @param token_data [Object] # @return [Boolean] def valid_token_data?(token_data) @@ -91,6 +137,20 @@ def valid_token_data?(token_data) signature.is_a?(String) && !signature.empty? && valid_payload?(token_data[:p]) end + # @param wire [Hash, nil] + # @return [Html2rss::Web::SelectorsDocument, nil] + def selectors_from_wire(wire) + return if wire.nil? + + SelectorsDocument.from_wire(wire) + end + + # @param selectors [Html2rss::Web::SelectorsDocument, nil] + # @return [Boolean] + def signable_selectors?(selectors) + !selectors.nil? && !selectors.empty? + end + # @param payload [Object] # @return [Boolean] def valid_payload?(payload) @@ -98,7 +158,8 @@ def valid_payload?(payload) payload[:u].is_a?(String) && payload[:l].is_a?(String) && payload[:e].is_a?(Integer) && - (payload[:t].nil? || payload[:t].is_a?(String)) + (payload[:t].nil? || payload[:t].is_a?(String)) && + valid_selectors_wire?(payload[:c]) end end end @@ -113,14 +174,16 @@ class << self # @param url [String] # @param secret_key [String] # @param strategy [String, nil] + # @param selectors [Html2rss::Web::SelectorsDocument, nil] # @param expires_in [Integer] # @return [Html2rss::Web::FeedToken, nil] - def create(username:, url:, secret_key:, strategy: nil, expires_in: DEFAULT_EXPIRY) - return unless valid_inputs?(username, url, secret_key, strategy) + # rubocop:disable-next Metrics/ParameterLists + def create(username:, url:, secret_key:, strategy: nil, selectors: nil, expires_in: DEFAULT_EXPIRY) + return unless valid_inputs?(username, url, secret_key, strategy, selectors) expires_at = Time.now.to_i + expires_in.to_i - signature = sign(secret_key, signature_payload(username, url, expires_at, strategy)) - FeedToken.new(username:, url:, expires_at:, signature:, strategy:) + signature = sign(secret_key, signature_payload(username, url, expires_at, strategy, selectors)) + FeedToken.new(username:, url:, expires_at:, signature:, strategy:, selectors:) end # @param encoded_token [String, nil] @@ -153,7 +216,8 @@ def valid_signature?(token, secret_key) return false unless secret_key.is_a?(String) && !secret_key.empty? expected_signature = sign(secret_key, signature_payload( - token.username, token.url, token.expires_at, token.strategy + token.username, token.url, token.expires_at, + token.strategy, token.selectors )) signatures_match?(token.signature, expected_signature) end @@ -164,10 +228,12 @@ def valid_signature?(token, secret_key) # @param url [String] # @param expires_at [Integer] # @param strategy [String, nil] + # @param selectors [Html2rss::Web::SelectorsDocument, nil] # @return [Hash{Symbol=>Object}] - def signature_payload(username, url, expires_at, strategy) + def signature_payload(username, url, expires_at, strategy, selectors = nil) payload = { username:, url:, expires_at: } payload[:strategy] = strategy if strategy + payload[:selectors] = selectors.to_wire if selectors && !selectors.empty? payload end @@ -192,10 +258,11 @@ def signatures_match?(first, second) # @param url [Object] # @param secret_key [Object] # @param strategy [Object] + # @param selectors [Object] # @return [Boolean] - def valid_inputs?(username, url, secret_key, strategy) + def valid_inputs?(username, url, secret_key, strategy, selectors = nil) valid_username?(username) && UrlValidator.valid_url?(url) && valid_secret_key?(secret_key) && - valid_strategy?(strategy) + valid_strategy?(strategy) && valid_selectors?(selectors) end # @param username [Object] @@ -218,6 +285,12 @@ def valid_strategy?(strategy) strategy.is_a?(String) && !strategy.empty? && strategy.length <= 50 && strategy.match?(/\A[a-z0-9_]+\z/) end + + # @param selectors [Object] + # @return [Boolean] + def valid_selectors?(selectors) + selectors.nil? || selectors.is_a?(SelectorsDocument) + end end end end diff --git a/app/web/security/log_sanitizer.rb b/app/web/security/log_sanitizer.rb index 7c403ddde..5d656adbf 100644 --- a/app/web/security/log_sanitizer.rb +++ b/app/web/security/log_sanitizer.rb @@ -12,12 +12,18 @@ module LogSanitizer PATH_KEYS = Set['endpoint', 'path'].freeze class << self + # Redacts the feed-token segment while keeping named sub-routes readable, so + # studio calls stay attributable in the audit trail. + # # @param path [String, nil] # @return [String, nil] def sanitize_path(path) return if path.nil? - path.to_s.gsub(FEED_TOKEN_ROUTE, '\1[REDACTED]\3') + path.to_s.gsub(FEED_TOKEN_ROUTE) do + prefix, segment, extension = Regexp.last_match.captures + named_feed_route?(segment) ? Regexp.last_match(0) : "#{prefix}[REDACTED]#{extension}" + end end # @param details [Hash] @@ -30,6 +36,15 @@ def sanitize_details(details) private + # Route names are owned by the router; asking it keeps one list of + # non-token segments under +/api/v1/feeds+. + # + # @param segment [String, nil] + # @return [Boolean] + def named_feed_route?(segment) + Routes::ApiV1::FeedRoutes::STUDIO_POSTS.key?(segment) + end + # @param key [Object] # @param value [Object] # @return [Object] diff --git a/app/web/security/selectors_document.rb b/app/web/security/selectors_document.rb new file mode 100644 index 000000000..e94f51370 --- /dev/null +++ b/app/web/security/selectors_document.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +require 'json' + +module Html2rss + module Web + ## + # Client-authored selectors subtree after the root key gate and gem schema + # validation. May be signed into a feed token. + # + # This is the only owner of which config root keys may be client-authored. + # {.from_client} and {.from_wire} share one rule so a leaked signing + # secret cannot widen what decode will execute. + SelectorsDocument = Data.define(:selectors) do + ## + # @param selectors [Hash{Symbol=>Object}] selectors subtree + # @return [Html2rss::Web::SelectorsDocument] + def initialize(selectors:) + super(selectors: deep_freeze(selectors)) + end + + ## + # Fragment merged into the token generator config. Never includes channel, + # headers, or any other client-controlled root key. + # + # @return [Hash{Symbol=>Object}] + def to_config_fragment = { selectors: }.freeze + + ## + # Wire document stored under the token +c:+ key and covered by the signature. + # + # @return [Hash{Symbol=>Object}] + def to_wire = to_config_fragment + + ## + # @return [Boolean] + def empty? = selectors.empty? + + private + + # @param value [Object] + # @return [Object] + def deep_freeze(value) + case value + in Hash then value.each_value { deep_freeze(it) } + in Array then value.each { deep_freeze(it) } + # HashUtil reuses string references; dup so freezing the handoff cannot + # freeze the caller's mutable strings. + in String then return value.dup.freeze + else return value + end + value.freeze + end + end + + class SelectorsDocument + ## + # Raised when a fragment clears the root key gate but fails gem schema validation. + # + # Carries the report so callers can render issues without re-validating or + # inspecting the exception message. + class SchemaInvalid < BadRequestError + # @return [Html2rss::Config::ValidationReport] + attr_reader :report + + # @param report [Html2rss::Config::ValidationReport] + def initialize(report) + @report = report + super(report.to_s) + end + end + + # Root keys a client fragment may contain. Published schema reads this set. + CLIENT_ROOT_KEYS = Set[:selectors].freeze + # Root keys rejected by name so the error is not a generic invalid-config. + DENIED_ROOT_KEYS = Set[ + :auto_source, :channel, :directory, :headers, :params, :registry, :request, :strategy, :stylesheets + ].freeze + # Keeps the signed token inside practical feed-URL lengths. + MAX_WIRE_BYTES = 2_048 + # Placeholder channel used only so schema validation can judge the selectors subtree. + VALIDATION_CHANNEL_URL = 'https://example.com/' + private_constant :VALIDATION_CHANNEL_URL + + class << self + ## + # Builds a selectors document from a client config fragment. + # + # @param hash [Hash] client object; only +selectors+ is permitted + # @return [Html2rss::Web::SelectorsDocument] + # @raise [Html2rss::Web::BadRequestError] when the fragment fails the root key gate + def from_client(hash) = build(hash, validate_shape: true) + + ## + # Rebuilds a selectors document from a decoded but *unverified* token wire document. + # + # Runs the structural root key gate only. Schema validation is withheld until the + # HMAC has been checked, so an unauthenticated caller cannot drive the gem + # validator with an unsigned token. {.from_verified_wire} runs it afterwards. + # + # @param hash [Hash] wire object previously returned by {#to_wire} + # @return [Html2rss::Web::SelectorsDocument] + # @raise [Html2rss::Web::BadRequestError] when the fragment fails the root key gate + def from_wire(hash) = build(hash, validate_shape: false) + + ## + # Rebuilds a selectors document from a wire document whose signature already verified. + # + # @param hash [Hash] wire object previously returned by {#to_wire} + # @return [Html2rss::Web::SelectorsDocument] + # @raise [Html2rss::Web::BadRequestError] when the fragment fails the root key gate + def from_verified_wire(hash) = build(hash, validate_shape: true) + + private + + # @param hash [Object] + # @param validate_shape [Boolean] + # @return [Html2rss::Web::SelectorsDocument] + def build(hash, validate_shape:) + selectors = permitted_selectors(normalized_fragment(hash)) + enforce_wire_limit!(selectors) + validate_shape!(selectors) if validate_shape + new(selectors:) + end + + # @param hash [Object] + # @return [Hash{Symbol=>Object}] + def normalized_fragment(hash) + raise BadRequestError, 'selectors document must be an object' unless hash.is_a?(Hash) + + Html2rss::HashUtil.deep_symbolize_keys(hash, context: 'selectors document') + end + + # @param fragment [Hash{Symbol=>Object}] + # @return [Hash{Symbol=>Object}] + def permitted_selectors(fragment) + reject_keys!(fragment) + selectors = fragment[:selectors] + raise BadRequestError, 'selectors must be an object' unless selectors.is_a?(Hash) + + selectors + end + + # @param fragment [Hash{Symbol=>Object}] + # @return [void] + def reject_keys!(fragment) + denied = fragment.each_key.find { DENIED_ROOT_KEYS.include?(it) } + raise BadRequestError, "#{denied} is not allowed" if denied + + unknown = fragment.each_key.find { !CLIENT_ROOT_KEYS.include?(it) } + raise BadRequestError, "#{unknown} is not allowed" if unknown + end + + # @param selectors [Hash{Symbol=>Object}] + # @return [void] + def enforce_wire_limit!(selectors) + return if JSON.generate({ selectors: }).bytesize <= MAX_WIRE_BYTES + + raise BadRequestError, "selectors document exceeds #{MAX_WIRE_BYTES} bytes" + end + + # @param selectors [Hash{Symbol=>Object}] + # @return [void] + # @raise [Html2rss::Web::SelectorsDocument::SchemaInvalid] + def validate_shape!(selectors) + report = Html2rss::Config.validate({ channel: { url: VALIDATION_CHANNEL_URL }, selectors: }) + return if report.success? + + raise SchemaInvalid, report + end + end + end + end +end diff --git a/bin/dev b/bin/dev index a65b8bf1f..1d1c68a17 100755 --- a/bin/dev +++ b/bin/dev @@ -3,16 +3,31 @@ set -e +# Caller wins (make up sets the host loopback URL). .env must not clobber it. +preset_botasaurus_url="${BOTASAURUS_SCRAPER_URL-}" + # Load .env if exists if [ -f .env ]; then export $(cat .env | grep -v '^#' | xargs) fi +if [ -n "$preset_botasaurus_url" ]; then + export BOTASAURUS_SCRAPER_URL="$preset_botasaurus_url" +fi + export RACK_ENV=${RACK_ENV:-development} -# BusyBox lsof in this image reports internal FDs and exits 0, so prefer netstat. +# BusyBox lsof reports internal FDs and exits 0, so the container keeps Linux netstat. +# macOS netstat has no -l listening flag, so the same check never sees port 4000. port_in_use() { - netstat -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "[:.]$1$" + case "$(uname -s)" in + Darwin) + lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1 + ;; + *) + netstat -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "[:.]$1$" + ;; + esac } # Cleanup function for graceful shutdown diff --git a/docker-compose.botasaurus.yml b/docker-compose.botasaurus.yml new file mode 100644 index 000000000..6ee840bb7 --- /dev/null +++ b/docker-compose.botasaurus.yml @@ -0,0 +1,11 @@ +# Published scraper for local `make up`. The web app stays on the host via bin/dev. +services: + botasaurus: + image: html2rss/botasaurus-scrape-api:latest + restart: unless-stopped + environment: + SENTRY_DSN: ${BOTASAURUS_SENTRY_DSN:-} + SCRAPE_TIMEOUT_SECONDS: 45 + SCRAPE_WORK_TIMEOUT_SECONDS: 30 + ports: + - "127.0.0.1:4010:4010" diff --git a/docker-compose.yml b/docker-compose.yml index 2590d9376..703d6356d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: HTML2RSS_SECRET_KEY: ${HTML2RSS_SECRET_KEY:?set HTML2RSS_SECRET_KEY in .env (generate with `openssl rand -hex 32`)} HTML2RSS_ACCESS_TOKEN: ${HTML2RSS_ACCESS_TOKEN:?set HTML2RSS_ACCESS_TOKEN in .env (generate with `openssl rand -hex 32`)} AUTO_SOURCE_ENABLED: ${AUTO_SOURCE_ENABLED:-true} + # Config refinement studio (validate/preview/suggest_selectors); token-gated, off by default. + STUDIO_ENABLED: ${STUDIO_ENABLED:-false} HEALTH_CHECK_TOKEN: ${HEALTH_CHECK_TOKEN:-} # Web Sentry (Project A): SENTRY_DSN; logs opt-in via SENTRY_ENABLE_LOGS=true SENTRY_DSN: ${SENTRY_DSN:-} diff --git a/docs/README.md b/docs/README.md index fb3f4cd2e..5bf85eefd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ Welcome! This is the canonical source of truth for contributing to `html2rss-web - **Start here for contributors**: This document. - **Architecture & Request Lifecycle**: [docs/architecture.md](architecture.md) - **UI/Design rules**: [docs/design-system.md](design-system.md) +- **Personas & surface ownership**: org workspace [`docs/personas.md`](../../docs/personas.md) (not this repo) - **Agent execution constraints**: [AGENTS.md](../AGENTS.md) - **Generated contract artifacts**: `public/openapi.yaml` - **Public-facing intro**: [README.md](../README.md) @@ -38,28 +39,30 @@ Running the app directly on the host is not supported. ### Common Commands (Inside Dev Container) -| Command | Purpose | -| ------------------------------ | ---------------------------------------------------------- | -| `make setup` | Install Ruby and Node dependencies. | -| `make dev` | Run Ruby (port 4000) and frontend (port 4001) dev servers. | -| `make ready` | Pre-commit gate: `make quick-check` + `bundle exec rspec`. | +| Command | Purpose | +| ------------------------------ | -------------------------------------------------------------------------- | +| `make setup` | Install Ruby and Node dependencies. | +| `make dev` | Run Ruby (port 4000) and frontend (port 4001) dev servers. | +| `make ready` | Pre-commit gate: `make quick-check` + `bundle exec rspec`. | | `make ci-ready` | CI parity gate: `make ready` + `make openapi-verify` + frontend e2e smoke. | -| `make test` | Run Ruby and frontend test suites. | -| `make lint` | Run all linters. | -| `make yard-verify-public-docs` | Enforce typed YARD docs for public methods in `app/`. | -| `make openapi` | Regenerate `public/openapi.yaml` from request specs. | -| `make openapi-verify` | Verify generated OpenAPI and frontend client artifacts are current. | -| `make openapi-lint` | Lint OpenAPI with Redocly + Spectral. | +| `make test` | Run Ruby and frontend test suites. | +| `make lint` | Run all linters. | +| `make yard-verify-public-docs` | Enforce typed YARD docs for public methods in `app/`. | +| `make openapi` | Regenerate `public/openapi.yaml` from request specs. | +| `make openapi-verify` | Verify generated OpenAPI and frontend client artifacts are current. | +| `make openapi-lint` | Lint OpenAPI with Redocly + Spectral. | ### Frontend pnpm Scripts -| Command | Purpose | -| ----------------------- | -------------------------------------------- | -| `pnpm run dev` | Vite dev server with hot reload (port 4001). | -| `pnpm run build` | Build static assets into `frontend/dist/`. | -| `pnpm run lint` | Run ESLint across the frontend workspace. | -| `pnpm run test:run` | Unit tests (Vitest). | -| `pnpm run test:contract`| Contract tests with MSW. | +| Command | Purpose | +| ------------------------ | -------------------------------------------- | +| `pnpm run dev` | Vite dev server with hot reload (port 4001). | +| `pnpm run build` | Build static assets into `frontend/dist/`. | +| `pnpm run lint` | Run ESLint across the frontend workspace. | +| `pnpm run test:run` | All Vitest projects (Node unit + jsdom integration). | +| `pnpm run test:unit` | Node project: parsers, routes, drafts, storage, apiHttp. | +| `pnpm run test:integration` | jsdom project: App, ConfigStudio, ResultDisplay, timer hooks. | +| `pnpm run test:e2e` | Playwright smoke (one worker). | Development routing defaults: @@ -76,7 +79,7 @@ To change or add API endpoints, follow this sequence: 1. **Ruby Request Spec**: Define the new behavior or endpoint in `spec/html2rss/web/app_integration_spec.rb` or a dedicated request spec. 2. **OpenAPI Generation**: Run `make openapi` inside the Dev Container to regenerate `public/openapi.yaml` from the spec metadata. 3. **Verify Contract**: Run `make openapi-verify` and `make openapi-lint` to ensure the generated file matches the specs and is valid. -4. **Frontend Client**: Keep generated client artifacts in `frontend/src/api/generated` aligned with `public/openapi.yaml`. +4. **Frontend Client**: Regenerate with `make openapi-client` (`frontend/openapi-ts.config.ts`). Do not hand-edit `frontend/src/api/generated/**`. Only `frontend/src/api/http/*.ts` may value-import that tree; parsers own domain types after unwrap. JSON Feed preview stays `fetch(json_public_url)` and is not the SDK. Always verify the contract before committing API changes. @@ -100,12 +103,13 @@ make ci-ready ### Testing Layers -| Layer | Tooling | Focus | -| ----------------- | ------------------------ | ---------------------------------------------------- | -| Ruby API | RSpec + Rack::Test | Feed creation, retrieval, auth paths. | -| Frontend unit | Vitest + Testing Library | Component rendering and hooks with mocked fetch. | -| Frontend contract | Vitest + MSW | End-to-end fetch flows against mocked API responses. | -| Docker smoke | RSpec (`:docker`) | Net::HTTP probes against the containerised service. | +| Layer | Tooling | Focus | +| ---------------------- | ------------------------ | ---------------------------------------------------- | +| Ruby API | RSpec + Rack::Test | Feed creation, retrieval, auth paths. | +| Frontend unit | Vitest (Node) | Parsers, routes, drafts, storage, apiHttp adapters. | +| Frontend integration | Vitest (jsdom) + MSW | App and ConfigStudio with real hooks; timer hooks. | +| Frontend e2e | Playwright | Stubbed SPA smoke: create/token/result/refine journeys. | +| Docker smoke | RSpec (`:docker`) | Net::HTTP probes against the containerised service. | --- @@ -140,7 +144,7 @@ make ci-ready - Regenerate with `make openapi`. - Drift must fail with `make openapi-verify`. - Quality must fail with `make openapi-lint`. -- Frontend generated client code under `frontend/src/api/generated` is machine-generated only. +- Frontend generated client code under `frontend/src/api/generated` is machine-generated only (`make openapi-client`). Value-import it only from `frontend/src/api/http/*.ts`. The client `baseUrl` is `/api/v1` (same-origin); public OpenAPI `servers` stay unchanged. Domain types live in parsers after unwrap. JSON Feed preview is `fetch(json_public_url)`, not `renderFeedByToken`. --- @@ -177,6 +181,7 @@ Search these pages for examples, plugins, and configuration options: - **Frontend Style**: - Follow visual and CSS rules in [design-system.md](design-system.md). - Use Preact components in `frontend/src/`. + - SPA JSON transport lives in `frontend/src/api/http/` (generated client + unwrap). UI modules must not value-import `frontend/src/api/generated`. - Use shared styles in `public/shared-ui.css` or app-specific styles in `frontend/src/styles/`. - Do not modify `frontend/dist/` directly. - **Testing**: @@ -189,14 +194,15 @@ Search these pages for examples, plugins, and configuration options: Managed flags and environment keys: -| Name | Env key | Type | Default | -| --------------------------------- | --------------------------------- | -------------- | ---------------------------------------- | -| `auto_source_enabled` | `AUTO_SOURCE_ENABLED` | boolean | `true` in development/test, else `false` | -| `health_check_token` | `HEALTH_CHECK_TOKEN` | string | `nil` | -| `build_tag` | `BUILD_TAG` | string | `unknown` outside production | -| `git_sha` | `GIT_SHA` | string | `unknown` outside production | -| `sentry_dsn` | `SENTRY_DSN` | string | `nil` | -| `sentry_enable_logs` | `SENTRY_ENABLE_LOGS` | boolean | `false` | +| Name | Env key | Type | Default | +| --------------------- | --------------------- | ------- | ---------------------------------------- | +| `auto_source_enabled` | `AUTO_SOURCE_ENABLED` | boolean | `true` in development/test, else `false` | +| `studio_enabled` | `STUDIO_ENABLED` | boolean | `true` in development/test, else `false` | +| `health_check_token` | `HEALTH_CHECK_TOKEN` | string | `nil` | +| `build_tag` | `BUILD_TAG` | string | `unknown` outside production | +| `git_sha` | `GIT_SHA` | string | `unknown` outside production | +| `sentry_dsn` | `SENTRY_DSN` | string | `nil` | +| `sentry_enable_logs` | `SENTRY_ENABLE_LOGS` | boolean | `false` | Rules: @@ -206,6 +212,10 @@ Rules: - `BUILD_TAG` and `GIT_SHA` are required in production so startup logs can identify the deployed build. - Add or change flags in code, tests, and this table together. +`STUDIO_ENABLED=true` exposes the config refinement studio: `POST /api/v1/feeds/validate`, `POST /api/v1/feeds/preview`, and `POST /api/v1/feeds/suggest_selectors`. All three require a valid access token and return `403 Studio is disabled` when the flag is off. `POST /api/v1/feeds` returns the same `403` when the body carries a `selectors` object, because those selectors are studio output; a body without `selectors` stays the automatic path and keeps working with the flag off. `preview` and `suggest_selectors` additionally need `AUTO_SOURCE_ENABLED=true` because they fetch the caller's URL live, and they only accept URLs the caller's account is allowed to reach. Validate accepts the same page URL and, when it is present and allowed, exports it as `channel.url` beside the selectors. Authenticated preview samples are capped at 10 items and include only safe HTTP(S) image URLs. `GET /api/v1` advertises `instance.studio.enabled` so the SPA can hide selector controls when the flag is off. Refinement stays on `#/result/:token`. Leave the flag off in production unless you intend to let token holders run live extraction against arbitrary allowed URLs. + +Rate limiting still uses the shared `RATE_LIMIT_MAX_REQUESTS` budget over `RATE_LIMIT_WINDOW_SECONDS`. `POST /api/v1/feeds`, `POST /api/v1/feeds/preview`, and `POST /api/v1/feeds/suggest_selectors` each consume five request slots because they run live extraction. Validate stays at one slot. Admission counts the current size and then charges the cost, so a budget of N can overrun by cost − 1. A successful preview page is kept for the process lifetime in `PreviewPageCache` and reused for the same URL until a different URL replaces the slot or the process exits (which removes the temp file). This fixed weighting protects the more expensive HTTP/browser extraction paths without adding another deployment flag. + --- ## Observability Contract @@ -218,9 +228,9 @@ Critical-path event families: auth, feed create, feed render, request errors. Use separate Sentry projects for html2rss-web and botasaurus-scrape-api. Never share a DSN. -| Env var | Service | Project | -| --- | --- | --- | -| `SENTRY_DSN` | html2rss-web | A (web) | +| Env var | Service | Project | +| ----------------------- | --------------------- | ----------- | +| `SENTRY_DSN` | html2rss-web | A (web) | | `BOTASAURUS_SENTRY_DSN` | botasaurus-scrape-api | B (scraper) | Compose maps `BOTASAURUS_SENTRY_DSN` into the botasaurus service as optional (`${BOTASAURUS_SENTRY_DSN:-}`). It does not fall back to web `SENTRY_DSN`. Leave it unset until you want scraper error reporting. @@ -229,14 +239,14 @@ Compose maps `BOTASAURUS_SENTRY_DSN` into the botasaurus service as optional (`$ Default `docker-compose.yml` aligns botasaurus-scrape-api, the html2rss gem client, and html2rss-web so the scraper exhausts its budget before the web tier aborts the request. Keep **scrape total (45) ≤ feed build (50) ≤ Falcon (55)**; the **work** budget (30) applies only after the browser is ready on the scraper. -| Variable | Service | Default | Role | -| --- | --- | --- | --- | -| `SCRAPE_TIMEOUT_SECONDS` | botasaurus-scrape-api | `45` | Handler wall (queue, boot, navigate, wait) | -| `SCRAPE_WORK_TIMEOUT_SECONDS` | botasaurus-scrape-api | `30` | Post-boot navigate, selector wait, scroll | -| `BOTASAURUS_SCRAPE_TIMEOUT_SECONDS` | html2rss (web) | `45` | HTTPX POST `/scrape` cap (mirrors scrape total) | -| `BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS` | html2rss | `30` | Max `wait_timeout_seconds` in feed YAML | -| `HTML2RSS_TOTAL_TIMEOUT_SECONDS` | html2rss-web | `50` | Feed build budget (scrape + extraction) | -| `REQUEST_TIMEOUT_SECONDS` | html2rss-web | `55` | Falcon server request timeout | +| Variable | Service | Default | Role | +| ---------------------------------------- | --------------------- | ------- | ----------------------------------------------- | +| `SCRAPE_TIMEOUT_SECONDS` | botasaurus-scrape-api | `45` | Handler wall (queue, boot, navigate, wait) | +| `SCRAPE_WORK_TIMEOUT_SECONDS` | botasaurus-scrape-api | `30` | Post-boot navigate, selector wait, scroll | +| `BOTASAURUS_SCRAPE_TIMEOUT_SECONDS` | html2rss (web) | `45` | HTTPX POST `/scrape` cap (mirrors scrape total) | +| `BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS` | html2rss | `30` | Max `wait_timeout_seconds` in feed YAML | +| `HTML2RSS_TOTAL_TIMEOUT_SECONDS` | html2rss-web | `50` | Feed build budget (scrape + extraction) | +| `REQUEST_TIMEOUT_SECONDS` | html2rss-web | `55` | Falcon server request timeout | When triaging `GATEWAY_TIMEOUT`, capacity-shaped `SERVICE_UNAVAILABLE` (queue/boot), or `error_category:timeout`, confirm both projects use this ladder. Scraper terminal timeouts near **45s** with web failures near **50–55s** indicate aligned budgets; scraper failures near **20–25s** while web waits longer usually mean stale `SCRAPE_*` / `BOTASAURUS_*` env on one side. Split queue/boot timeouts from work timeouts before blaming the ladder (see **Alert baselines**). @@ -260,6 +270,7 @@ Start triage from the newest `feed.create`, `feed.render`, and `request.error` e - `GATEWAY_TIMEOUT` / `timeout` with `timeout_phase` `work` (or nil transport hop) → slow or hostile target, or timeout ladder mismatch (see **Compose timeout ladder**) Outer scraper deadlines reclaim Chromium via WorkLease (session force-close) so worker slots free before the HTTP 504 returns; lingering `timeout_phase:queue` under light traffic after deploy usually means reclaim or per-host admit saturation (`SCRAPE_MAX_PER_HOST`), not soft-cancel theater. + ### Alert baselines After baseline traffic, configure per project: diff --git a/docs/design-system.md b/docs/design-system.md index ca3a8723e..cd03ffabc 100644 --- a/docs/design-system.md +++ b/docs/design-system.md @@ -38,31 +38,35 @@ If a page looks like it came from a different product, the change is wrong even ## Journey Grammar (enforced) -- **Create:** URL field is the task. Visiting `#/create` or `#!/create` remounts create; hashbang aliases canonicalize to `#/create`. Bare create does not auto-submit. When the typed query finds catalog entries (URL equivalence or text substring), show matching feeds as a subordinate list under the field — never a second primary task or in-app catalog browser. On an empty Create URL, up to three Feed Directory starters may appear as subordinate chrome (lean: `.ui-eyebrow` + list + demoted `utility-link` escape under the list when creation is enabled; Notice when creation is disabled). Escape and utility-strip share **Browse Feed Directory (N)** (`catalogEntries.length`); hide starters and the under-list escape while typing, when find has hits, or while creating. -- **Token gate:** a native `` over the still-mounted, inert URL task (one interactive task). Auth copy is in-field (`tokenError`); ActionFeedback stays on create. Access Token persists until Logout with no storage UI. -- **Result:** primary CTA is **Copy feed URL**. Open feed / JSON / feed-reader are demoted secondary actions and stay available while preview loads. Preview is non-blocking confirmation only. -- **Unmatched result:** `#/result/:token` is valid only with a matching in-memory result. Missing or mismatched tokens recover onto remounted `#/create` (no API rehydrate, no failure chrome, no durable shareable result page). -- **Status eyebrow:** one vocabulary (`Feed ready`) on the result header; preview progress lives in the preview section. -- **Focus:** create autofocuses the URL field; token autofocuses the token field; result focuses the Copy control. +- **Create:** URL field is the task. Visiting `#/create` or `#!/create` remounts create; hashbang aliases canonicalize to `#/create`. Bare create does not auto-submit. When the typed query finds catalog entries (URL equivalence or text substring), show matching feeds as a subordinate list under the field — never a second primary task or in-app catalog browser. On an empty Create URL, up to three Feed Directory starters may appear as subordinate chrome (lean: `.ui-eyebrow` + list + demoted `utility-link` escape under the list when creation is enabled; Notice when creation is disabled). Escape and utility-strip share **Browse Feed Directory (N)** (`catalogEntries.length`); hide starters and the under-list escape while typing, when find has hits, or while creating. Create busy stays in-control (`DominantField`); no loading `Notice` under the field. +- **Token gate:** a native `` over the still-mounted, inert URL task (one interactive task). Auth copy is in-field (`tokenError`); ActionFeedback stays on create for failure only. Access Token persists until Logout with no storage UI. +- **Ready (member):** one result frame (`data-state` `ready`). Primary CTA is **Copy feed URL** (initial focus). Open feed / JSON / feed-reader and **Create another feed** are demoted. Guest results stay compact: at most 5 items, no images, no selector controls. Authenticated studio-enabled results mount the flat editor immediately (zero clicks to reveal). **Save and generate feed** stays disabled and demoted until the draft changes, then becomes the sole `btn--primary`. Focus stays on Copy on ready entry. When studio is disabled or there is no token, the editor stays hidden. There is no `#/refine` route. +- **Unresolved:** when create returns `EXTRACTION_EMPTY`, studio is enabled, and an access token is present, the same result frame opens with the flat editor already mounted (`data-state` `unresolved`). Listing URL is the subject; wire `Decision#message` is the single ResultDisplay notice; **Save and generate feed** is the only primary immediately (no edit gate). No **Feed ready**, no **Copy feed URL**, no directory handoff. Demoted **Create another feed** sits below the studio workspace. Hash is in-memory `#/result` (no feed token). Empty extraction with studio off or no token stays a create error (**Try again** when retryable). +- **Flat editor:** Items stays visible (proportional mono input, not `input--lg`). Optional Title / Link / Published appear only with selector, candidate, issue, or manual evidence. Field + chips are one `.studio-field` unit; chips use `COPY.choicesFor` only (no bare field-name eyebrow). Inline Enhance (no card). One `.studio-actions` row → `.ui-item-list--grid` meadow (2-col wide / 1-col below `48rem`). Suggest status uses reserved-height `.studio-suggestion-status`; suggest/save must not re-echo the unresolved Decision notice. No Refine toggle, Advanced expander, field-status panels, or View-as-YAML disclosure. +- **Utilities:** **Copy YAML** copies the current channel URL + selectors document in one click (no expandable preview). Ready-only **Open Feed Directory issue** opens a prefilled html2rss-configs GitHub issue (channel URL + selectors YAML + blank `directory.title` / `directory.topics`); never includes access or feed tokens. Native-feed Notice may appear above Save when the preview reports one. +- **Known limits:** On authenticated studio-enabled results, validate and suggest run eagerly on mount; the first live preview still waits for an edit so the create-time meadow stays single. Edits keep the previous meadow visible and mark it loading until live studio output replaces it — one list only; no return to create-time meadow once a studio sample exists. Until that sample exists (including first preview load), keep the create-time meadow and show **neutral** labels on visible optional fields — found/missing metadata appears only after the first studio sample. Studio meadow caption (page already fetched) only when sample items are present — no filler chrome when the meadow is empty. Member previews may show up to 10 items and safe HTTP(S) images. Unmatched `#/result/:token` / bare `#/result` without matching in-memory state recover onto remounted `#/create` (no API rehydrate, no durable shareable result page). Invalid URL and auth stay on create / token. +- **Status eyebrow:** ready uses `Feed ready`; unresolved uses `URL` with the listing as the title; preview progress lives in the preview section. +- **Focus:** create autofocuses the URL field; token autofocuses the token field; ready focuses the Copy control; unresolved keeps Save and generate as the primary task (no Copy). ### Vocabulary -| Job | Words | -| --- | --- | -| Input | **URL** | -| Output | **Feed URL** | -| Create | **Create feed** / **Creating feed** | -| Result | **Feed ready** / **Copy feed URL** | -| Retry | **Try again** (button only) | -| Preview | **Checking preview** / **Check again** | -| Catalog | **Feed Directory** | -| Token | **Access token** | +| Job | Words | +| ------- | ------------------------------------------------------------------------------------------------------------------ | +| Input | **URL** | +| Output | **Feed URL** | +| Create | **Create feed** / **Creating feed** | +| Result | **Feed ready** / **Copy feed URL** | +| Studio | **Headline choices** / **Add** / **Remove** / **Save and generate feed** / **Copy YAML** / **Open Feed Directory issue** | +| Retry | **Try again** (button only) | +| Preview | **Checking preview** / **Check again** / **Based on the page already fetched for this URL** (with items only) | +| Catalog | **Feed Directory** | +| Token | **Access token** | ## Non-Negotiable Surface Rules - Background must use the same dark canvas and top-light treatment defined in `shared-ui.css`. - Shared cards must use the same border, radius, and surface treatment via `.ui-card` (plus modifiers). No page-local framed-surface forks. -- Shared feed/preview items must use `.ui-item-list` / `.ui-item` / `.ui-item__*` grammar. +- Shared feed/preview items must use `.ui-item-list` / `.ui-item` / `.ui-item__*` grammar. Meadow previews compose `.ui-item-list--grid` (2 columns on wide rail; collapses to 1 column below `48rem`). - Serif display typography is reserved for major titles (`.ui-display-title`) and the wordmark. - Sans UI typography is the default for controls, supporting copy, and metadata. - Mono is reserved for URLs, tokens, and machine-like values. @@ -173,6 +177,7 @@ Prefer composing these primitives before inventing new classes: - `ui-eyebrow` - `ui-eyebrow--ghost` - `ui-item-list` +- `ui-item-list--grid` - `ui-item` - `ui-item__meta` - `ui-item__title` (link child when URL present) @@ -294,6 +299,8 @@ These are common signs that the system is drifting: - semantic states are encoded as a growing list of presentational classes - result actions gated on preview loading - ActionFeedback stacked under the token dialog +- create loading Notice under the URL field (create busy belongs in DominantField only) +- unresolved Decision notice echoed again by studio save or suggest failure - token gate replacing the URL composer instead of a dialog over inert create If you see one of these, consolidate instead of layering more CSS. diff --git a/frontend/e2e/fixtures.ts b/frontend/e2e/fixtures.ts new file mode 100644 index 000000000..690fafa60 --- /dev/null +++ b/frontend/e2e/fixtures.ts @@ -0,0 +1,116 @@ +import { test as base, expect, type Page, type Request } from '@playwright/test'; +import { COPY } from '../src/journey/copy'; +import { + createdFeedBody, + emptyCatalog, + metadataBody, + pageUrlFrom, + selectorsFromValidateRequest, + studioPreviewSuccessBody, + suggestSuccessBody, + validateSuccessBody, + validateYamlForSelectors, + type MetadataOptions, +} from '../src/__tests__/mocks/apiFixtures'; + +export { expect } from '@playwright/test'; + +export const API_ROOT = /\/api\/v1\/?$/; +export const API_CONFIGS = /\/api\/v1\/configs\/?$/; +export const API_FEEDS = /\/api\/v1\/feeds\/?$/; +export const API_VALIDATE = /\/api\/v1\/feeds\/validate\/?$/; +export const API_SUGGEST = /\/api\/v1\/feeds\/suggest_selectors\/?$/; +export const API_STUDIO_PREVIEW = /\/api\/v1\/feeds\/preview\/?$/; +export const API_JSON_PREVIEW = /\/api\/v1\/feeds\/(?:generated-token|studio-saved-token)\.json$/; + +export const EXAMPLE_URL = 'https://example.com/articles'; + +type SpaFixtures = { + fulfillJson: (path: RegExp, body: unknown, status?: number, contentType?: string) => Promise; + stubSpaApi: (options?: MetadataOptions) => Promise; + stubStudioEndpoints: () => Promise; + createReadyResult: ( + options?: MetadataOptions & { stubFeedCreates?: (page: Page) => Promise } + ) => Promise; +}; + +export const test = base.extend({ + fulfillJson: async ({ page }, use) => { + await use(async (path, body, status = 200, contentType = 'application/json') => { + await page.route(path, async (route) => { + await route.fulfill({ + status, + contentType, + body: JSON.stringify(body), + }); + }); + }); + }, + + stubSpaApi: async ({ fulfillJson }, use) => { + await use(async (options: MetadataOptions = {}) => { + const isCatalogEnabled = options.isCatalogEnabled ?? true; + await fulfillJson(API_ROOT, metadataBody(options)); + if (isCatalogEnabled) await fulfillJson(API_CONFIGS, emptyCatalog); + }); + }, + + stubStudioEndpoints: async ({ page, fulfillJson }, use) => { + await use(async (options: { suggestBody?: unknown } = {}) => { + await page.route(API_VALIDATE, async (route) => { + const posted = route.request().postDataJSON() as unknown; + const selectors = selectorsFromValidateRequest(posted); + const channelUrl = pageUrlFrom(posted); + const body = validateSuccessBody(selectors, validateYamlForSelectors(selectors, channelUrl)); + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(body), + }); + }); + await fulfillJson( + API_SUGGEST, + options.suggestBody ?? + suggestSuccessBody({ + items: [{ selector: 'article', enhance: false, sample: 'Sample headline' }], + title: [{ selector: 'h2', sample: 'Sample headline' }], + }) + ); + await fulfillJson(API_STUDIO_PREVIEW, studioPreviewSuccessBody()); + }); + }, + + createReadyResult: async ({ page, fulfillJson, stubSpaApi, stubStudioEndpoints }, use) => { + await use(async (options = {}) => { + const { stubFeedCreates, ...metadataOptions } = options; + await stubSpaApi({ isCatalogEnabled: false, ...metadataOptions }); + if (metadataOptions.isStudioEnabled !== false) await stubStudioEndpoints(); + if (stubFeedCreates) { + await stubFeedCreates(page); + } else { + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + } + await fulfillJson(API_JSON_PREVIEW, { items: [] }, 200, 'application/feed+json'); + await page.addInitScript(() => { + localStorage.setItem('html2rss_access_token', 'token-123'); + }); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + }); + }, +}); + +export function isSameOriginApi(url: string, path: RegExp): boolean { + const requested = new URL(url); + return path.test(requested.pathname) && requested.host !== 'api.html2rss.dev'; +} + +export function collectPosts(page: Page, path: RegExp, bag: Request[]) { + page.on('request', (request) => { + if (request.method() === 'POST' && isSameOriginApi(request.url(), path)) { + bag.push(request); + } + }); +} diff --git a/frontend/e2e/smoke.spec.ts b/frontend/e2e/smoke.spec.ts index 51c318684..c9850d048 100644 --- a/frontend/e2e/smoke.spec.ts +++ b/frontend/e2e/smoke.spec.ts @@ -1,107 +1,178 @@ -import { expect, test } from '@playwright/test'; +import { type Request } from '@playwright/test'; import { COPY } from '../src/journey/copy'; +import { + buildStructuredErrorResponse, + createdFeedBody, + jsonPreviewItems, + suggestSuccessBody, +} from '../src/__tests__/mocks/apiFixtures'; +import { + API_CONFIGS, + API_FEEDS, + API_JSON_PREVIEW, + API_SUGGEST, + EXAMPLE_URL, + expect, + isSameOriginApi, + test, +} from './fixtures'; + +const EMPTY_NOTICE = 'We could not extract feed items from this page yet.'; +const READY_URL = 'https://example.com/ready-articles'; +const ACCESS_TOKEN = 'token-123'; +const FEED_TOKEN = 'generated-token'; + +const VIEWPORTS = [ + { name: 'desktop', width: 1280, height: 720 }, + { name: 'mobile', width: 390, height: 844 }, +] as const; + +async function assertNoRefineChrome(page: import('@playwright/test').Page) { + await expect(page.locator('details')).toHaveCount(0); + await expect(page.locator('[class*="refine-"]')).toHaveCount(0); +} test.describe('frontend smoke', () => { - test('loads create flow and inline access-token gate', async ({ page }) => { - await page.route(/\/api\/v1$/, async (route) => { + test('covers one member journey from token gate through ready studio', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + fulfillJson, + }) => { + await stubSpaApi({ isCatalogEnabled: false }); + await stubStudioEndpoints(); + await page.route(API_FEEDS, async (route) => { + if (route.request().method() !== 'POST') { + await route.fallback(); + return; + } + const body = route.request().postDataJSON() as { url?: string }; + if (body.url === READY_URL) { + await route.fulfill({ + status: 201, + contentType: 'application/json', + body: JSON.stringify(createdFeedBody({ url: READY_URL, feed_token: FEED_TOKEN })), + }); + return; + } await route.fulfill({ - status: 200, + status: 422, contentType: 'application/json', - body: JSON.stringify({ - success: true, - data: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: true, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - }), + body: JSON.stringify( + buildStructuredErrorResponse({ + code: 'EXTRACTION_EMPTY', + message: EMPTY_NOTICE, + kind: 'input', + retryable: false, + next_action: 'correct_input', + retry_action: 'none', + }) + ), }); }); + await fulfillJson(API_JSON_PREVIEW, jsonPreviewItems, 200, 'application/feed+json'); + + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByRole('heading', { name: COPY.tokenTitle })).toBeVisible(); + await page.locator('#access-token').fill(ACCESS_TOKEN); + await page.getByRole('button', { name: COPY.saveAndContinue }).click(); + + await expect(page).toHaveURL(/\/#\/result$/); + await expect(page.locator('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + await expect(page.getByText(EMPTY_NOTICE)).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toBeVisible(); + await expect(page.getByText(COPY.feedReady)).toHaveCount(0); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toHaveCount(0); + await expect(page.getByRole('link', { name: COPY.proposeDirectory })).toHaveCount(0); + + await page.getByRole('button', { name: COPY.createAnother }).click(); + await expect(page).toHaveURL(/\/#\/create/); + const urlField = page.getByLabel(COPY.urlLabel); + await urlField.fill(READY_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + await expect(page.locator('.result-shell')).toHaveAttribute('data-state', 'ready'); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeFocused(); + await expect(page.getByLabel(COPY.itemsSelector, { exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass(/btn--ghost/); + await expect( + page.getByRole('button', { name: COPY.addSelector('Sample headline') }).first() + ).toBeVisible(); + await assertNoRefineChrome(page); + const proposal = page.getByRole('link', { name: COPY.proposeDirectory }); + await expect(proposal).toBeVisible(); + const href = (await proposal.getAttribute('href')) ?? ''; + expect(href).toContain('https://github.com/html2rss/html2rss-configs/issues/new'); + const body = new URL(href).searchParams.get('body') ?? ''; + expect(body).toContain(READY_URL); + expect(body).not.toContain(ACCESS_TOKEN); + expect(body).not.toContain(FEED_TOKEN); + }); + + test('loads create flow and inline access-token gate', async ({ page, stubSpaApi }) => { + const catalogUrls: string[] = []; + page.on('request', (request) => { + if (API_CONFIGS.test(new URL(request.url()).pathname)) catalogUrls.push(request.url()); + }); + await stubSpaApi(); await page.goto('/'); await expect(page.getByLabel(COPY.urlLabel)).toBeVisible(); await expect(page.getByRole('button', { name: COPY.createFeed })).toBeVisible(); + await expect.poll(() => catalogUrls.length, { timeout: 5000 }).toBeGreaterThan(0); + expect(catalogUrls.every((url) => isSameOriginApi(url, API_CONFIGS))).toBe(true); - await page.getByLabel(COPY.urlLabel).fill('https://example.com/articles'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); await page.getByRole('button', { name: COPY.createFeed }).click(); await expect(page.getByRole('heading', { name: COPY.tokenTitle })).toBeVisible(); - await expect(page.getByRole('textbox', { name: COPY.tokenTitle })).toBeVisible(); - await expect(page.getByRole('button', { name: COPY.saveAndContinue })).toBeVisible(); - await expect(page.getByRole('button', { name: COPY.back })).toBeVisible(); await expect(page.locator('dialog')).toHaveAttribute('open'); + await expect(page.locator('#access-token')).toBeFocused(); await expect(page.getByLabel(COPY.urlLabel)).toHaveCount(1); - await expect(page.getByText(COPY.createFailedTitle, { exact: true })).toHaveCount(0); - await expect(page.getByText(COPY.createFailedRetryTitle, { exact: true })).toHaveCount(0); + expect( + await page.getByLabel(COPY.urlLabel).evaluate((element) => element.closest('[inert]') !== null) + ).toBe(true); await page.getByRole('button', { name: COPY.back }).click(); await expect(page).toHaveURL(/#\/create(?:\?.*)?$/); await expect(page.getByRole('button', { name: COPY.createFeed })).toBeVisible(); - await expect(page.locator('.form-shell')).toHaveAttribute('data-state', 'create'); - await expect(page.getByLabel(COPY.utilities)).toBeVisible(); }); - test('remounts create from BrandLockup and hashbang and clears creation chrome', async ({ page }) => { - await page.route(/\/api\/v1$/, async (route) => { - await route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ - success: true, - data: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: false, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - }), - }); - }); - await page.route(/\/api\/v1\/feeds$/, async (route) => { - await route.fulfill({ - status: 500, - contentType: 'application/json', - body: JSON.stringify({ - success: false, - error: { - kind: 'server', - code: 'INTERNAL_SERVER_ERROR', - retryable: true, - next_action: 'retry', - retry_action: 'primary', - message: 'Access denied', - }, - }), - }); - }); + test('remounts create from BrandLockup and hashbang and clears creation chrome', async ({ + page, + stubSpaApi, + fulfillJson, + }) => { + await stubSpaApi({ isTokenRequired: false }); + await fulfillJson( + API_FEEDS, + { + success: false, + error: { + kind: 'server', + code: 'INTERNAL_SERVER_ERROR', + retryable: true, + next_action: 'retry', + retry_action: 'primary', + message: 'Access denied', + }, + }, + 500 + ); await page.goto('/#/create'); - await page.getByLabel(COPY.urlLabel).fill('https://example.com/articles'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); await page.getByRole('button', { name: COPY.createFeed }).click(); await expect(page.getByText(COPY.createFailedRetryTitle, { exact: true })).toBeVisible(); await page.getByRole('link', { name: 'html2rss' }).click(); await expect(page.getByText(COPY.createFailedRetryTitle, { exact: true })).toHaveCount(0); - await expect(page.locator('.form-shell')).toHaveAttribute('data-state', 'create'); await expect(page.getByLabel(COPY.urlLabel)).toBeFocused(); await page.getByRole('button', { name: COPY.createFeed }).click(); @@ -112,100 +183,237 @@ test.describe('frontend smoke', () => { }); await expect(page).toHaveURL(/\/#\/create$/); await expect(page.getByText(COPY.createFailedRetryTitle, { exact: true })).toHaveCount(0); - await expect(page.locator('.form-shell')).toHaveAttribute('data-state', 'create'); await expect(page.getByLabel(COPY.urlLabel)).toBeFocused(); }); - test('shows result after successful feed creation and recovers unmatched result routes onto create', async ({ - page, - }) => { - await page.route(/\/api\/v1$/, async (route) => { - await route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ - success: true, - data: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: true, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - }), - }); - }); + test('recovers empty result and retired refine hashes onto create', async ({ page, stubSpaApi }) => { + await stubSpaApi({ isCatalogEnabled: false }); - await page.route(/\/api\/v1\/feeds$/, async (route) => { - await route.fulfill({ - status: 201, - contentType: 'application/json', - body: JSON.stringify({ - success: true, - data: { - feed: { - id: 'feed-123', - name: 'Example Feed', - url: 'https://example.com/articles', - feed_token: 'generated-token', - public_url: '/api/v1/feeds/generated-token', - json_public_url: '/api/v1/feeds/generated-token.json', - created_at: '2026-04-05T08:59:00.000Z', - updated_at: '2026-04-05T09:00:00.000Z', - }, - }, - }), - }); - }); + await page.goto('/#/result/missing-token'); + await expect(page).toHaveURL(/\/#\/create$/); + await expect(page.getByLabel(COPY.urlLabel)).toBeVisible(); + await expect(page.getByText('Saved result unavailable')).toHaveCount(0); - await page.route(/\/api\/v1\/feeds\/generated-token\.json$/, async (route) => { - await route.fulfill({ - status: 200, - contentType: 'application/feed+json', - body: JSON.stringify({ - items: [ - { - title: 'Sample preview item', - content_text: 'Current preview fetch includes rendered content.', - date_published: '2026-04-05T09:00:00.000Z', - url: 'https://example.com/articles/sample-preview-item', - }, - ], - }), - }); - }); + await page.goto('/#/refine'); + await expect(page).toHaveURL(/\/#\/create$/); + await expect(page.getByLabel(COPY.urlLabel)).toBeVisible(); + }); + + test('realigns a stale ready result hash without leaving the workspace', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + fulfillJson, + }) => { + await stubSpaApi(); + await stubStudioEndpoints(); + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + await fulfillJson(API_JSON_PREVIEW, jsonPreviewItems, 200, 'application/feed+json'); await page.addInitScript(() => { localStorage.setItem('html2rss_access_token', 'token-123'); }); await page.goto('/'); - await page.getByLabel(COPY.urlLabel).fill('https://example.com/articles'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + + const created = page.waitForRequest((request: Request) => { + if (request.method() !== 'POST' || !isSameOriginApi(request.url(), API_FEEDS)) return false; + return request.headers().authorization === 'Bearer token-123'; + }); await page.getByRole('button', { name: COPY.createFeed }).click(); + await created; await expect(page.getByText(COPY.feedReady)).toBeVisible(); - await expect(page.locator('.result-shell')).toHaveAttribute('data-state', 'result'); - await expect(page.getByText('Example Feed')).toBeVisible(); - await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeVisible(); - await expect(page.getByRole('link', { name: COPY.openFeed })).toBeVisible(); - await expect(page.getByRole('link', { name: COPY.openJsonFeed })).toBeVisible(); - await expect(page.getByRole('link', { name: COPY.openInFeedReader })).toBeVisible(); - await expect(page.getByRole('button', { name: COPY.createAnother })).toBeVisible(); - await expect(page.getByText('Sample preview item')).toBeVisible(); - await expect(page.getByText('Current preview fetch includes rendered content.')).toBeVisible(); + await expect(page.locator('.result-shell')).toHaveAttribute('data-state', 'ready'); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeFocused(); + await expect(page.locator('.ui-item-list')).toHaveCount(1); await page.goto('/#/result/missing-token'); - - await expect(page).toHaveURL(/\/#\/create$/); - await expect(page.getByLabel(COPY.urlLabel)).toBeVisible(); - await expect(page.getByText('Saved result unavailable')).toHaveCount(0); - await expect(page.locator('.result-recovery')).toHaveCount(0); + await expect(page).toHaveURL(/\/#\/result\/generated-token$/); + await expect(page.getByText(COPY.feedReady)).toBeVisible(); }); + + for (const viewport of VIEWPORTS) { + test.describe(`compact result matrix @ ${viewport.name}`, () => { + test.beforeEach(async ({ page }) => { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); + }); + + test('candidate success keeps Copy-feed primary and one meadow', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + fulfillJson, + }) => { + await stubSpaApi({ isCatalogEnabled: false }); + await stubStudioEndpoints(); + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + await fulfillJson(API_JSON_PREVIEW, jsonPreviewItems, 200, 'application/feed+json'); + await page.addInitScript(() => { + localStorage.setItem('html2rss_access_token', 'token-123'); + }); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeFocused(); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass(/btn--ghost/); + await expect(page.getByRole('button', { name: COPY.copyYaml })).toBeVisible(); + await expect(page.getByRole('link', { name: COPY.proposeDirectory })).toBeVisible(); + await expect(page.locator('.ui-item-list')).toHaveCount(1); + await expect( + page.getByRole('button', { name: COPY.addSelector('Sample headline') }).first() + ).toBeVisible(); + await expect(page.getByLabel(COPY.fieldLink)).toHaveCount(0); + await assertNoRefineChrome(page); + }); + + test('valid-empty suggestions stay neutral with manual activators', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + fulfillJson, + }) => { + await stubSpaApi({ isCatalogEnabled: false }); + await stubStudioEndpoints({ suggestBody: suggestSuccessBody() }); + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + await fulfillJson(API_JSON_PREVIEW, { items: [] }, 200, 'application/feed+json'); + await page.addInitScript(() => { + localStorage.setItem('html2rss_access_token', 'token-123'); + }); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + await expect(page.getByText(COPY.selectorSuggestionsEmpty)).toBeVisible(); + await expect( + page.getByRole('button', { name: COPY.addOptionalField(COPY.fieldTitle) }) + ).toBeVisible(); + await expect(page.getByLabel(COPY.fieldTitle)).toHaveCount(0); + await assertNoRefineChrome(page); + }); + + test('suggestion failure stays retryable without faking candidates', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + fulfillJson, + }) => { + await stubSpaApi({ isCatalogEnabled: false }); + await stubStudioEndpoints(); + await page.route(API_SUGGEST, async (route) => { + await route.fulfill({ + status: 403, + contentType: 'application/json', + body: JSON.stringify( + buildStructuredErrorResponse({ + code: 'FORBIDDEN', + message: 'Auto source feature is disabled', + kind: 'auth', + retryable: false, + next_action: 'none', + retry_action: 'none', + }) + ), + }); + }); + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + await fulfillJson(API_JSON_PREVIEW, { items: [] }, 200, 'application/feed+json'); + await page.addInitScript(() => { + localStorage.setItem('html2rss_access_token', 'token-123'); + }); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + await expect(page.getByText('Auto source feature is disabled')).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.retrySelectorSuggestions })).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.addSelector('Sample headline') })).toHaveCount(0); + await assertNoRefineChrome(page); + }); + + test('guest ready has no studio controls', async ({ page, stubSpaApi, fulfillJson }) => { + await stubSpaApi({ isCatalogEnabled: false, isTokenRequired: false }); + await fulfillJson(API_FEEDS, createdFeedBody(), 201); + await fulfillJson(API_JSON_PREVIEW, jsonPreviewItems, 200, 'application/feed+json'); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeFocused(); + await expect(page.getByLabel(COPY.itemsSelector, { exact: true })).toHaveCount(0); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toHaveCount(0); + await assertNoRefineChrome(page); + }); + + test('authenticated ready promotes Save only after edit', async ({ page, createReadyResult }) => { + await createReadyResult({ isCatalogEnabled: false }); + + const save = page.getByRole('button', { name: COPY.saveAndGenerate }); + await expect(save).toHaveClass(/btn--ghost/); + await expect(save).toBeDisabled(); + await page.getByLabel(COPY.itemsSelector, { exact: true }).fill('article.card'); + await expect(save).toHaveClass(/btn--primary/); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeVisible(); + await assertNoRefineChrome(page); + }); + + test('unresolved mounts Save as the sole primary', async ({ + page, + stubSpaApi, + stubStudioEndpoints, + }) => { + await stubSpaApi({ isCatalogEnabled: false }); + await stubStudioEndpoints(); + await page.route(API_FEEDS, async (route) => { + if (route.request().method() !== 'POST') { + await route.fallback(); + return; + } + await route.fulfill({ + status: 422, + contentType: 'application/json', + body: JSON.stringify( + buildStructuredErrorResponse({ + code: 'EXTRACTION_EMPTY', + message: EMPTY_NOTICE, + kind: 'input', + retryable: false, + next_action: 'correct_input', + retry_action: 'none', + }) + ), + }); + }); + await page.addInitScript(() => { + localStorage.setItem('html2rss_access_token', 'token-123'); + }); + await page.goto('/'); + await page.getByLabel(COPY.urlLabel).fill(EXAMPLE_URL); + await page.getByRole('button', { name: COPY.createFeed }).click(); + + await expect(page).toHaveURL(/\/#\/result$/); + await expect(page.locator('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass(/btn--primary/); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toHaveCount(0); + await expect(page.getByRole('link', { name: COPY.proposeDirectory })).toHaveCount(0); + await expect(page.locator('.ui-item-list')).toHaveCount(0); + await assertNoRefineChrome(page); + }); + + test('studio-off hides the editor on ready', async ({ page, createReadyResult }) => { + await createReadyResult({ isCatalogEnabled: false, isStudioEnabled: false }); + + await expect(page.getByText(COPY.feedReady)).toBeVisible(); + await expect(page.getByRole('button', { name: COPY.copyFeedUrl })).toBeFocused(); + await expect(page.getByLabel(COPY.itemsSelector, { exact: true })).toHaveCount(0); + await expect(page.getByRole('button', { name: COPY.saveAndGenerate })).toHaveCount(0); + await assertNoRefineChrome(page); + }); + }); + } }); diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 932c2b50f..2b0a552c4 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -21,7 +21,6 @@ export default tseslint.config( globals: { ...globals.browser, ...globals.node, - ...globals.vitest, }, }, plugins: { @@ -33,6 +32,31 @@ export default tseslint.config( '@typescript-eslint/no-explicit-any': 'off', 'unicorn/filename-case': 'off', 'unicorn/better-regex': 'warn', + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: ['**/api/generated', '**/api/generated/**'], + allowTypeImports: true, + message: 'Value-import the generated OpenAPI client only from src/api/http/*.ts.', + }, + ], + }, + ], + }, + }, + { + files: ['src/api/http/*.ts', 'src/__tests__/apiHttp*.test.ts'], + rules: { + 'no-restricted-imports': 'off', + }, + }, + { + files: ['e2e/**/*.ts'], + rules: { + // Playwright fixtures expose a `use` callback that is not a React Hook. + 'react-hooks/rules-of-hooks': 'off', }, } ); diff --git a/frontend/openapi-ts.config.ts b/frontend/openapi-ts.config.ts new file mode 100644 index 000000000..c2e019755 --- /dev/null +++ b/frontend/openapi-ts.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from '@hey-api/openapi-ts'; + +export default defineConfig({ + input: '../public/openapi.yaml', + output: 'src/api/generated', + plugins: [ + { + name: '@hey-api/client-fetch', + baseUrl: '/api/v1', + }, + ], +}); diff --git a/frontend/package.json b/frontend/package.json index d05984e21..3e55d8e11 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,13 +14,13 @@ "format": "prettier --write .", "format:check": "prettier --check .", "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", - "openapi:generate": "openapi-ts -i ../public/openapi.yaml -o src/api/generated -c @hey-api/client-fetch", - "openapi:verify": "openapi-ts -i ../public/openapi.yaml -o src/api/generated -c @hey-api/client-fetch && git diff --exit-code -- src/api/generated", + "openapi:generate": "openapi-ts", + "openapi:verify": "openapi-ts && git diff --exit-code -- src/api/generated", "test": "vitest", "test:run": "vitest run", - "test:unit": "vitest run --reporter=verbose --config vitest.unit.config.ts", - "test:contract": "vitest run --reporter=verbose --config vitest.contract.config.ts", - "test:ci": "pnpm run test:unit && pnpm run test:contract", + "test:unit": "vitest run --project unit --reporter=verbose", + "test:integration": "vitest run --project integration --reporter=verbose", + "test:ci": "pnpm run test:run", "test:e2e": "playwright test" }, "dependencies": { @@ -34,6 +34,7 @@ "@preact/preset-vite": "^2.10.6", "@testing-library/jest-dom": "^7.0.1", "@testing-library/preact": "^3.2.4", + "@testing-library/user-event": "14.6.7", "@typescript/native": "npm:typescript@^7.0.2", "baseline-browser-mapping": "^2.11.20", "eslint": "^10.9.1", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 81da0b8e2..0d2116d5d 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -16,10 +16,10 @@ function resolveChromiumExecutablePath(): string | undefined { export default defineConfig({ testDir: './e2e', - fullyParallel: true, + fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, - workers: process.env.CI ? 1 : undefined, + workers: 1, reporter: 'list', use: { baseURL: 'http://127.0.0.1:4001', diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 1ad0f73f3..07f870de8 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -6,7 +6,8 @@ settings: overrides: postcss@<8.5.10: '>=8.5.10' - js-yaml: ^4.3.1 + js-yaml: ^4.3.2 + colord: ^2.9.4 undici: ^7.28.0 brace-expansion: ^5.0.8 fast-uri: ^3.1.7 @@ -41,6 +42,9 @@ importers: '@testing-library/preact': specifier: ^3.2.4 version: 3.2.4(preact@10.29.8) + '@testing-library/user-event': + specifier: 14.6.7 + version: 14.6.7(@testing-library/dom@8.20.1) '@typescript/native': specifier: npm:typescript@^7.0.2 version: typescript@7.0.2 @@ -622,6 +626,12 @@ packages: peerDependencies: preact: '>=10 || ^10.0.0-alpha.0 || ^10.0.0-beta.0' + '@testing-library/user-event@14.6.7': + resolution: {integrity: sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==} + engines: {node: '>=12', npm: '>=6'} + peerDependencies: + '@testing-library/dom': '>=7.21.4' + '@types/aria-query@5.0.4': resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} @@ -1027,8 +1037,8 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - colord@2.9.3: - resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} + colord@2.10.0: + resolution: {integrity: sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==} confbox@0.2.4: resolution: {integrity: sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==} @@ -1646,8 +1656,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@4.3.1: - resolution: {integrity: sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==} + js-yaml@4.3.2: + resolution: {integrity: sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==} hasBin: true jsdom@30.0.1: @@ -2803,7 +2813,7 @@ snapshots: dependencies: '@jsdevtools/ono': 7.1.3 '@types/json-schema': 7.0.15 - js-yaml: 4.3.1 + js-yaml: 4.3.2 '@hey-api/openapi-ts@0.0.0-next-20260824173136': dependencies: @@ -3077,6 +3087,10 @@ snapshots: '@testing-library/dom': 8.20.1 preact: 10.29.8 + '@testing-library/user-event@14.6.7(@testing-library/dom@8.20.1)': + dependencies: + '@testing-library/dom': 8.20.1 + '@types/aria-query@5.0.4': {} '@types/chai@5.2.3': @@ -3461,7 +3475,7 @@ snapshots: color-name@1.1.4: {} - colord@2.9.3: {} + colord@2.10.0: {} confbox@0.2.4: {} @@ -3477,7 +3491,7 @@ snapshots: dependencies: env-paths: 2.2.1 import-fresh: 3.3.1 - js-yaml: 4.3.1 + js-yaml: 4.3.2 parse-json: 5.2.0 optionalDependencies: typescript: '@typescript/typescript6@6.0.2' @@ -4079,7 +4093,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@4.3.1: + js-yaml@4.3.2: dependencies: argparse: 2.0.1 @@ -4632,7 +4646,7 @@ snapshots: '@csstools/media-query-list-parser': 5.0.0(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0) '@csstools/selector-resolve-nested': 4.0.1(postcss-selector-parser@7.1.4) '@csstools/selector-specificity': 6.0.0(postcss-selector-parser@7.1.4) - colord: 2.9.3 + colord: 2.10.0 cosmiconfig: 9.0.2(@typescript/typescript6@6.0.2) css-functions-list: 3.3.3 css-tree: 3.2.1 diff --git a/frontend/pnpm-workspace.yaml b/frontend/pnpm-workspace.yaml index ca544cec8..51562709d 100644 --- a/frontend/pnpm-workspace.yaml +++ b/frontend/pnpm-workspace.yaml @@ -8,7 +8,8 @@ minimumReleaseAgeExclude: - brace-expansion@5.0.8 overrides: postcss@<8.5.10: ">=8.5.10" - js-yaml: "^4.3.1" + js-yaml: "^4.3.2" + colord: "^2.9.4" undici: "^7.28.0" brace-expansion: "^5.0.8" fast-uri: "^3.1.7" diff --git a/frontend/src/__tests__/App.contract.test.tsx b/frontend/src/__tests__/App.contract.test.tsx deleted file mode 100644 index e3ec55d4a..000000000 --- a/frontend/src/__tests__/App.contract.test.tsx +++ /dev/null @@ -1,153 +0,0 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { render, screen, fireEvent, waitFor } from '@testing-library/preact'; -import { http, HttpResponse } from 'msw'; -import { server, buildFeedResponse, buildStructuredErrorResponse } from './mocks/server'; -import { App } from '../components/App'; -import { COPY } from '../journey/copy'; - -describe('App contract', () => { - const token = 'contract-token'; - - beforeEach(() => { - history.replaceState({}, '', 'http://localhost:3000/#/create'); - localStorage.clear(); - sessionStorage.clear(); - localStorage.setItem('html2rss_access_token', token); - }); - - it('shows feed result when the API returns structured create payload and preview feed', async () => { - const nativeFetch = fetch; - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation((input, init) => { - if (String(input).endsWith('/api/v1/feeds/generated-token.json')) { - expect((init?.headers as Record | undefined)?.Accept).toBe('application/feed+json'); - return Promise.resolve( - Response.json( - { - items: [ - { - title: 'Contract Item', - content_text: 'Contract preview excerpt.', - url: 'https://example.com/contract-item', - date_published: '2024-01-01T00:00:00Z', - }, - ], - }, - { status: 200, headers: { 'Content-Type': 'application/feed+json' } } - ) - ); - } - - return nativeFetch(input, init); - }); - - server.use( - http.post('/api/v1/feeds', async ({ request }) => { - const body = (await request.json()) as { url: string }; - - expect(body).toEqual({ url: 'https://example.com/articles' }); - expect(request.headers.get('authorization')).toBe(`Bearer ${token}`); - - return HttpResponse.json( - buildFeedResponse({ - url: body.url, - feed_token: 'generated-token', - public_url: '/api/v1/feeds/generated-token', - json_public_url: '/api/v1/feeds/generated-token.json', - }), - { status: 201 } - ); - }), - http.get('http://localhost:3000/api/v1/feeds/generated-token.json', ({ request }) => { - expect(request.headers.get('accept')).toBe('application/feed+json'); - - return HttpResponse.json( - { - items: [ - { - title: 'Contract Item', - content_text: 'Contract preview excerpt.', - url: 'https://example.com/contract-item', - date_published: '2024-01-01T00:00:00Z', - }, - ], - }, - { - headers: { 'content-type': 'application/feed+json' }, - } - ); - }), - http.get('/api/v1/feeds/generated-token.json', ({ request }) => { - expect(request.headers.get('accept')).toBe('application/feed+json'); - - return HttpResponse.json({ - items: [ - { - title: 'Contract Item', - content_text: 'Contract preview excerpt.', - url: 'https://example.com/contract-item', - date_published: '2024-01-01T00:00:00Z', - }, - ], - }); - }) - ); - - render(); - - await waitFor(() => { - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - }); - expect(screen.queryByRole('combobox')).not.toBeInTheDocument(); - - const urlInput = screen.getByLabelText(COPY.urlLabel) as HTMLInputElement; - fireEvent.input(urlInput, { target: { value: 'https://example.com/articles' } }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await waitFor(() => { - expect(screen.getByText(COPY.feedReady)).toBeInTheDocument(); - expect(screen.getByText('Example Feed')).toBeInTheDocument(); - expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'result'); - expect(screen.getByLabelText(COPY.feedUrl)).toBeInTheDocument(); - expect(screen.getByRole('button', { name: COPY.copyFeedUrl })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: COPY.createAnother })).toBeInTheDocument(); - expect(screen.getByText(COPY.previewItemCount(1))).toBeInTheDocument(); - }); - fetchSpy.mockRestore(); - }); - - it('reopens the token gate when a saved token is rejected by structured auth metadata', async () => { - server.use( - http.post('/api/v1/feeds', async () => - HttpResponse.json( - buildStructuredErrorResponse({ - code: 'UNAUTHORIZED', - message: 'Authentication required', - kind: 'auth', - retryable: false, - next_action: 'enter_token', - retry_action: 'none', - }), - { status: 401 } - ) - ) - ); - - render(); - - await waitFor(() => { - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - }); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await screen.findByText(COPY.tokenRejected); - - expect(screen.getByRole('heading', { name: COPY.tokenTitle })).toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - expect(localStorage.getItem('html2rss_access_token')).toBeNull(); - expect(sessionStorage.getItem('html2rss_access_token')).toBeNull(); - }); -}); diff --git a/frontend/src/__tests__/App.integration.test.tsx b/frontend/src/__tests__/App.integration.test.tsx new file mode 100644 index 000000000..ddd781c4d --- /dev/null +++ b/frontend/src/__tests__/App.integration.test.tsx @@ -0,0 +1,614 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { render, screen, waitFor, within, fireEvent, cleanup } from '@testing-library/preact'; +import userEvent from '@testing-library/user-event'; +import { http, HttpResponse } from 'msw'; +import { server, buildFeedResponse, buildStructuredErrorResponse } from './mocks/server'; +import { + catalogBody, + catalogWireEntry, + JSON_FEED_PREVIEW_PATH, + jsonPreviewItems, + metadataBody, +} from './mocks/apiFixtures'; +import { App } from '../components/App'; +import { COPY } from '../journey/copy'; +import { resetAccessTokenMemory } from '../session/accessToken'; + +const EXAMPLE_URL = 'https://example.com/articles'; +const DIRECTORY_HREF = 'https://html2rss.github.io/feed-directory/#!url=http%3A%2F%2Flocalhost%3A3000%2F'; +const ACCESS_TOKEN = 'member-token'; + +const fao = catalogWireEntry({ + id: 'fao.org/newsroom', + channelUrl: 'https://www.fao.org/newsroom', + title: 'FAO Newsroom', +}); +const anthropic = catalogWireEntry({ + id: 'anthropic.com/news', + channelUrl: 'https://www.anthropic.com/news', + title: 'Anthropic — News', +}); +const bbcMundo = catalogWireEntry({ + id: 'bbc.com/mundo', + channelUrl: 'https://www.bbc.com/mundo', + title: 'BBC — Mundo', +}); +const bbcSounds = catalogWireEntry({ + id: 'bbc.co.uk/available_episodes', + path: '/bbc.co.uk/available_episodes.rss', + channelUrl: 'https://www.bbc.co.uk/programmes/%s/episodes/player', + title: 'BBC Sounds — Programme episodes', + parameterDefaults: { id: 'b006wkfp' }, +}); + +function useSuccessfulCreate(createCount?: { value: number }) { + server.use( + http.post('/api/v1/feeds', async ({ request }) => { + if (createCount) createCount.value += 1; + const body = (await request.json()) as { url: string; selectors?: unknown }; + return HttpResponse.json( + buildFeedResponse({ + url: body.url, + feed_token: 'generated-token', + public_url: '/api/v1/feeds/generated-token', + json_public_url: '/api/v1/feeds/generated-token.json', + }), + { status: 201 } + ); + }) + ); +} + +const EMPTY_EXTRACTION_NOTICE = 'We could not extract feed items from this page yet.'; + +function extractionEmptyResponse(message = EMPTY_EXTRACTION_NOTICE) { + return HttpResponse.json( + buildStructuredErrorResponse({ + code: 'EXTRACTION_EMPTY', + message, + kind: 'input', + retryable: false, + next_action: 'correct_input', + retry_action: 'none', + }), + { status: 422 } + ); +} + +async function createWithUrl(user: Awaited>, url = EXAMPLE_URL) { + await user.type(screen.getByLabelText(COPY.urlLabel), url); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); +} + +async function renderApp(user = userEvent.setup()) { + render(); + await screen.findByLabelText(COPY.urlLabel); + return user; +} + +describe('App', () => { + beforeEach(() => { + history.replaceState({}, '', 'http://localhost:3000/#/create'); + localStorage.clear(); + sessionStorage.clear(); + resetAccessTokenMemory(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('shows feed result when create and JSON preview succeed', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + const pending = Promise.withResolvers(); + server.use( + http.post('/api/v1/feeds', () => pending.promise), + http.get(JSON_FEED_PREVIEW_PATH, ({ request }) => { + expect(request.headers.get('accept')).toBe('application/feed+json'); + return HttpResponse.json(jsonPreviewItems, { + headers: { 'content-type': 'application/feed+json' }, + }); + }) + ); + + const user = await renderApp(); + expect(screen.queryByRole('combobox')).not.toBeInTheDocument(); + + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + expect(screen.getByRole('button', { name: COPY.creating })).toBeInTheDocument(); + expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'submitting'); + expect(document.querySelector('.notice[data-state="loading"]')).toBeNull(); + + pending.resolve( + HttpResponse.json( + buildFeedResponse({ + url: EXAMPLE_URL, + feed_token: 'generated-token', + public_url: '/api/v1/feeds/generated-token', + json_public_url: '/api/v1/feeds/generated-token.json', + }), + { status: 201 } + ) + ); + + await screen.findByText(COPY.feedReady); + expect(screen.getByText('Example Feed')).toBeInTheDocument(); + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'ready'); + await waitFor(() => { + expect(screen.getByText(COPY.previewItemCount(1))).toBeInTheDocument(); + expect(screen.getByText('Sample preview item')).toBeInTheDocument(); + }); + }); + + it('shows lean starters and hides them once the URL is non-empty', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use(http.get('/api/v1/configs', () => HttpResponse.json(catalogBody([fao], [fao.id])))); + + const user = await renderApp(); + + await screen.findByRole('link', { name: 'FAO Newsroom' }); + expect(screen.getByRole('list', { name: COPY.feedDirectory })).toBeInTheDocument(); + expect(screen.queryByText(COPY.feedDirectoryIntro)).not.toBeInTheDocument(); + + await user.type(screen.getByLabelText(COPY.urlLabel), 'example.com'); + expect(screen.queryByRole('link', { name: 'FAO Newsroom' })).not.toBeInTheDocument(); + expect(screen.queryByRole('list', { name: COPY.feedDirectory })).not.toBeInTheDocument(); + }); + + it('opens the active catalog hit with ArrowDown then Enter without creating', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + const createCount = { value: 0 }; + useSuccessfulCreate(createCount); + server.use(http.get('/api/v1/configs', () => HttpResponse.json(catalogBody([bbcMundo, bbcSounds])))); + + const user = await renderApp(); + await waitFor(() => { + expect(screen.getAllByRole('link', { name: COPY.browseFeedDirectory(2) }).length).toBeGreaterThan(0); + }); + const urlField = screen.getByLabelText(COPY.urlLabel); + fireEvent.input(urlField, { target: { value: 'bbc' } }); + await screen.findByRole('listbox', { name: COPY.catalogFindHitsLabel }); + + urlField.focus(); + await user.keyboard('{ArrowDown}'); + const active = screen.getByRole('option', { name: 'BBC Sounds — Programme episodes' }); + expect(active).toHaveAttribute('aria-selected', 'true'); + expect(active).toHaveAttribute('href', '/bbc.co.uk/available_episodes.rss?id=b006wkfp'); + expect(urlField).toHaveAttribute('aria-activedescendant', active.id); + + await user.keyboard('{Enter}'); + + expect(createCount.value).toBe(0); + }); + + it('does not double-create when Enter key-repeats', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + const createCount = { value: 0 }; + useSuccessfulCreate(createCount); + + await renderApp(); + const urlField = screen.getByLabelText(COPY.urlLabel); + fireEvent.input(urlField, { target: { value: EXAMPLE_URL } }); + fireEvent.keyDown(urlField, { key: 'Enter' }); + fireEvent.keyDown(urlField, { key: 'Enter', repeat: true }); + fireEvent.keyDown(urlField, { key: 'Enter', repeat: true }); + + await screen.findByText(COPY.feedReady); + expect(createCount.value).toBe(1); + }); + + it('surfaces retryable create errors with Decision copy', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + message: 'Access denied', + kind: 'server', + retryable: true, + next_action: 'retry', + retry_action: 'primary', + }), + { status: 500 } + ) + ) + ); + + const user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + await screen.findByText(COPY.createFailedRetryTitle); + expect(screen.getByText('Access denied')).toBeInTheDocument(); + }); + + it('opens unresolved on EXTRACTION_EMPTY with studio; stays create error when studio is off', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use(http.post('/api/v1/feeds', () => extractionEmptyResponse())); + + let user = await renderApp(); + await createWithUrl(user); + + await waitFor(() => { + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + }); + expect(location.hash).toBe('#/result'); + expect(screen.getByText(EXAMPLE_URL)).toBeInTheDocument(); + expect(screen.getByText(EMPTY_EXTRACTION_NOTICE)).toBeInTheDocument(); + expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); + expect(screen.getByLabelText(COPY.itemsSelector)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass('btn--primary'); + + cleanup(); + history.replaceState({}, '', 'http://localhost:3000/#/create'); + resetAccessTokenMemory(); + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use( + http.get(/\/api\/v1\/?$/, () => HttpResponse.json(metadataBody({ isStudioEnabled: false }))), + http.post('/api/v1/feeds', () => extractionEmptyResponse()) + ); + user = await renderApp(); + await createWithUrl(user); + + await screen.findByText(COPY.createFailedTitle); + expect(screen.getByText(EMPTY_EXTRACTION_NOTICE)).toBeInTheDocument(); + expect(document.querySelector('.result-shell')).not.toBeInTheDocument(); + expect(location.hash).toMatch(/#\/create/); + }); + + it('mints ready on unresolved save with items, or stays unresolved when save is still empty', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + const secondNotice = 'Still no items with these selectors.'; + let createCalls = 0; + server.use( + http.post('/api/v1/feeds', async ({ request }) => { + createCalls += 1; + const body = (await request.json()) as { url?: string; selectors?: unknown }; + if (!body.selectors) return extractionEmptyResponse(); + return extractionEmptyResponse(createCalls === 1 ? EMPTY_EXTRACTION_NOTICE : secondNotice); + }) + ); + + let user = await renderApp(); + await createWithUrl(user); + await waitFor(() => { + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + }); + expect(screen.getByText(EMPTY_EXTRACTION_NOTICE)).toBeInTheDocument(); + + let save = screen.getByRole('button', { name: COPY.saveAndGenerate }); + await waitFor(() => expect(save).toBeEnabled()); + await user.click(save); + + await waitFor(() => { + expect(screen.getAllByText(secondNotice)).toHaveLength(1); + }); + expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + expect(location.hash).toBe('#/result'); + expect(createCalls).toBeGreaterThanOrEqual(2); + + cleanup(); + history.replaceState({}, '', 'http://localhost:3000/#/create'); + resetAccessTokenMemory(); + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + createCalls = 0; + server.use( + http.post('/api/v1/feeds', async ({ request }) => { + createCalls += 1; + const body = (await request.json()) as { url?: string; selectors?: unknown }; + if (!body.selectors) return extractionEmptyResponse(); + return HttpResponse.json( + buildFeedResponse({ + url: body.url ?? EXAMPLE_URL, + feed_token: 'minted-token', + public_url: '/api/v1/feeds/minted-token', + json_public_url: '/api/v1/feeds/minted-token.json', + }), + { status: 201 } + ); + }), + http.get(JSON_FEED_PREVIEW_PATH, () => + HttpResponse.json(jsonPreviewItems, { + headers: { 'Content-Type': 'application/feed+json' }, + }) + ) + ); + user = await renderApp(); + await createWithUrl(user); + await waitFor(() => { + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + }); + save = screen.getByRole('button', { name: COPY.saveAndGenerate }); + await waitFor(() => expect(save).toBeEnabled()); + await user.click(save); + + await screen.findByText(COPY.feedReady); + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'ready'); + expect(location.hash).toBe('#/result/minted-token'); + expect(screen.getByRole('button', { name: COPY.copyFeedUrl })).toBeInTheDocument(); + expect(screen.getByLabelText(COPY.itemsSelector)).toBeInTheDocument(); + }); + + it('shows instance metadata failure as a banner without create-error chrome', async () => { + server.use(http.get(/\/api\/v1\/?$/, () => HttpResponse.json({ success: false }, { status: 503 }))); + + render(); + await screen.findByText(COPY.instanceMetadataUnavailable); + expect(screen.getByText(COPY.instanceUnavailable)).toBeInTheDocument(); + expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); + }); + + it('keeps utility order for guest and member, remaps OpenAPI, and places logout before Docker', async () => { + server.use( + http.get(/\/api\/v1\/?$/, () => + HttpResponse.json(metadataBody({ openapiUrl: 'http://127.0.0.1:4000/openapi.yaml' })) + ) + ); + + await renderApp(); + + const utilities = screen.getByLabelText(COPY.utilities); + expect( + [...utilities.querySelectorAll(':scope .utility-strip__items > a')].map((link) => link.textContent) + ).toEqual([ + COPY.browseFeedDirectory(), + COPY.bookmarkletTitle, + COPY.dockerInstall, + COPY.openapiSpec, + COPY.sourceCode, + ]); + expect(screen.getByRole('link', { name: COPY.openapiSpec })).toHaveAttribute( + 'href', + 'http://localhost:3000/openapi.yaml' + ); + expect(screen.getByRole('link', { name: COPY.browseFeedDirectory() })).toHaveAttribute( + 'href', + DIRECTORY_HREF + ); + + cleanup(); + history.replaceState({}, '', 'http://localhost:3000/#/create'); + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + await renderApp(); + await screen.findByRole('button', { name: COPY.logout }); + expect( + [ + ...screen + .getByLabelText(COPY.utilities) + .querySelectorAll(':scope .utility-strip__items > a, :scope .utility-strip__items > button'), + ].map((element) => element.textContent) + ).toEqual([ + COPY.browseFeedDirectory(), + COPY.bookmarkletTitle, + COPY.logout, + COPY.dockerInstall, + COPY.openapiSpec, + COPY.sourceCode, + ]); + }); + + it('clears the access token and remounts create on logout', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + + const user = await renderApp(); + await user.click(await screen.findByRole('button', { name: COPY.logout })); + + expect(localStorage.getItem('html2rss_access_token')).toBeNull(); + expect(sessionStorage.getItem('html2rss_access_token')).toBeNull(); + expect(location.hash).toMatch(/^#\/create/); + expect(screen.queryByRole('button', { name: COPY.logout })).not.toBeInTheDocument(); + expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); + }); + + it('prefills bookmarklet hash URLs over a saved draft', async () => { + localStorage.setItem( + 'html2rss_feed_draft_state', + JSON.stringify({ url: 'https://old.example.com/page' }) + ); + history.replaceState({}, '', 'http://localhost:3000/#/create?url=https%3A%2F%2Fexample.com%2Farticles'); + + await renderApp(); + await waitFor(() => { + expect(screen.getByLabelText(COPY.urlLabel)).toHaveValue(EXAMPLE_URL); + }); + expect(screen.getByRole('link', { name: COPY.bookmarkletTitle }).getAttribute('href')).toContain( + '/#/create?url=' + ); + }); + + it('saves a token from the prompt and resumes create', async () => { + useSuccessfulCreate(); + const user = await renderApp(); + + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + await screen.findByRole('heading', { name: COPY.tokenTitle }); + expect(location.hash).toMatch(/^#\/token/); + expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); + + await user.type(document.querySelector('#access-token') as HTMLInputElement, 'token-123'); + await user.click(screen.getByRole('button', { name: COPY.saveAndContinue })); + + await screen.findByText(COPY.feedReady); + expect(localStorage.getItem('html2rss_access_token')).toBe('token-123'); + expect(sessionStorage.getItem('html2rss_access_token')).toBeNull(); + }); + + it('keeps Decision chrome when token save create fails non-auth onto create', async () => { + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + message: 'Upstream failed', + kind: 'server', + retryable: true, + next_action: 'retry', + retry_action: 'primary', + }), + { status: 500 } + ) + ) + ); + + const user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + await screen.findByRole('heading', { name: COPY.tokenTitle }); + await user.type(document.querySelector('#access-token') as HTMLInputElement, 'token-123'); + await user.click(screen.getByRole('button', { name: COPY.saveAndContinue })); + + await waitFor(() => { + expect(location.hash).toMatch(/^#\/create/); + }); + await screen.findByText(COPY.createFailedRetryTitle); + expect(screen.getByText('Upstream failed')).toBeInTheDocument(); + expect(screen.queryByRole('heading', { name: COPY.tokenTitle })).not.toBeInTheDocument(); + expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'error'); + + // Remount from token→create must keep Decision error (hasAutoSubmitted blocks wipe/auto-retry). + await waitFor(() => { + expect(screen.getByText(COPY.createFailedRetryTitle)).toBeInTheDocument(); + expect(screen.getByText('Upstream failed')).toBeInTheDocument(); + expect(location.hash).toMatch(/^#\/create/); + expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'error'); + }); + }); + + it('reopens the token gate on auth rejection and ignores non-token forbidden', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none', + }), + { status: 401 } + ) + ) + ); + + let user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + await screen.findByText(COPY.tokenRejected); + expect(screen.getByRole('heading', { name: COPY.tokenTitle })).toBeInTheDocument(); + expect(localStorage.getItem('html2rss_access_token')).toBeNull(); + + cleanup(); + history.replaceState({}, '', 'http://localhost:3000/#/create'); + resetAccessTokenMemory(); + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + code: 'FORBIDDEN', + message: 'URL not allowed for this account', + kind: 'server', + retryable: false, + next_action: 'none', + retry_action: 'none', + }), + { status: 403 } + ) + ) + ); + user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + + await screen.findByText('URL not allowed for this account'); + expect(screen.queryByRole('heading', { name: COPY.tokenTitle })).not.toBeInTheDocument(); + expect(screen.queryByText(COPY.tokenRejected)).not.toBeInTheDocument(); + expect(localStorage.getItem('html2rss_access_token')).toBe(ACCESS_TOKEN); + }); + + it('clears creation chrome when remounting create via BrandLockup', async () => { + localStorage.setItem('html2rss_access_token', ACCESS_TOKEN); + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + message: 'Upstream failed', + kind: 'server', + retryable: true, + next_action: 'retry', + retry_action: 'primary', + }), + { status: 500 } + ) + ) + ); + + const user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), EXAMPLE_URL); + await user.click(screen.getByRole('button', { name: COPY.createFeed })); + await screen.findByText(COPY.createFailedRetryTitle); + + await user.click(screen.getByRole('link', { name: 'html2rss' })); + + await waitFor(() => { + expect(screen.queryByText(COPY.createFailedRetryTitle)).not.toBeInTheDocument(); + expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); + }); + expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); + }); + + it('promotes starters when feed creation is disabled', async () => { + server.use( + http.get(/\/api\/v1\/?$/, () => + HttpResponse.json({ + success: true, + data: { + api: { + name: 'html2rss-web API', + description: 'd', + openapi_url: 'https://example.test/openapi.yaml', + }, + instance: { + feed_creation: { enabled: false, access_token_required: false }, + studio: { enabled: true }, + catalog: { enabled: true, url: 'https://api.html2rss.dev/api/v1/configs' }, + }, + }, + }) + ), + http.get('/api/v1/configs', () => HttpResponse.json(catalogBody([fao], [fao.id]))) + ); + + await renderApp(); + await waitFor(() => { + expect(document.querySelector('.notice__title')?.textContent).toBe(COPY.feedDirectory); + }); + expect(screen.getByRole('link', { name: 'FAO Newsroom' })).toBeInTheDocument(); + expect(screen.getByText(COPY.creationDisabled)).toBeInTheDocument(); + }); + + it('lists catalog find hits for a text query', async () => { + server.use( + http.get('/api/v1/configs', () => HttpResponse.json(catalogBody([bbcMundo, bbcSounds, anthropic]))) + ); + + const user = await renderApp(); + await user.type(screen.getByLabelText(COPY.urlLabel), 'bbc'); + + await screen.findByRole('option', { name: 'BBC — Mundo' }); + expect(screen.getByRole('option', { name: 'BBC Sounds — Programme episodes' })).toHaveAttribute( + 'href', + '/bbc.co.uk/available_episodes.rss?id=b006wkfp' + ); + expect(screen.getByRole('listbox', { name: COPY.catalogFindHitsLabel })).toBeInTheDocument(); + expect(within(screen.getByRole('status')).getByText(COPY.catalogFindHint)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/__tests__/App.test.tsx b/frontend/src/__tests__/App.test.tsx deleted file mode 100644 index 3604dc021..000000000 --- a/frontend/src/__tests__/App.test.tsx +++ /dev/null @@ -1,1346 +0,0 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { render, screen, fireEvent, waitFor } from '@testing-library/preact'; -import type { FeedCreationError } from '../api/contracts'; -import { App } from '../components/App'; -import { COPY } from '../journey/copy'; - -vi.mock('../session/accessToken', () => ({ - useAccessToken: vi.fn(), - resetAccessTokenMemory: vi.fn(), -})); - -vi.mock('../feed/useFeedCreation', () => ({ - useFeedCreation: vi.fn(), -})); - -vi.mock('../hooks/useApiMetadata', () => ({ - useApiMetadata: vi.fn(), -})); - -vi.mock('../catalog/useCatalogEntries', () => ({ - useCatalogEntries: vi.fn(() => ({ entries: [], starters: [] })), -})); - -import { useAccessToken } from '../session/accessToken'; -import { useApiMetadata } from '../hooks/useApiMetadata'; -import { useFeedCreation } from '../feed/useFeedCreation'; -import { useCatalogEntries } from '../catalog/useCatalogEntries'; - -const mockUseAccessToken = useAccessToken as any; -const mockUseApiMetadata = useApiMetadata as any; -const mockUseFeedCreation = useFeedCreation as any; -const mockUseCatalogEntries = useCatalogEntries as any; - -const catalogHook = (entries: unknown[] = [], starters?: string[]) => ({ - entries, - starters: starters ?? entries.map((entry: any) => entry.id).slice(0, 3), -}); - -const mockCreatedFeedResult = { - feed: { - id: 'feed-123', - name: 'Example Feed', - url: 'https://example.com/articles', - feed_token: 'generated-token', - public_url: '/api/v1/feeds/generated-token', - json_public_url: '/api/v1/feeds/generated-token.json', - }, - preview: { - status: 'created' as const, - items: [], - isLoading: true, - }, - warnings: [], -}; - -async function expectCreateRemountedWithoutErrorChrome() { - await waitFor(() => { - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedRetryTitle)).not.toBeInTheDocument(); - }); - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); - await waitFor(() => { - expect(document.activeElement).toBe(screen.getByLabelText(COPY.urlLabel)); - }); -} - -describe('App', () => { - const mockSaveToken = vi.fn(); - const mockClearToken = vi.fn(); - const mockCreateFeed = vi.fn(); - const mockClearCreationError = vi.fn(); - const mockClearResult = vi.fn(); - const mockRetryPreviewFetch = vi.fn(); - /** Mirrors useFeedCreation: reject sets error; clearError clears it (App mock was static). */ - let creationHookError: FeedCreationError | undefined; - - beforeEach(() => { - vi.clearAllMocks(); - creationHookError = undefined; - history.replaceState({}, '', 'http://localhost:3000/#/create'); - localStorage.clear(); - mockCreateFeed.mockResolvedValue(mockCreatedFeedResult); - mockClearCreationError.mockImplementation(() => { - creationHookError = undefined; - }); - mockUseCatalogEntries.mockReturnValue(catalogHook()); - - mockUseAccessToken.mockReturnValue({ - token: undefined, - hasToken: false, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - mockUseApiMetadata.mockReturnValue({ - metadata: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: true, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - isLoading: false, - error: undefined, - }); - - mockUseFeedCreation.mockImplementation(() => ({ - isCreating: false, - result: undefined, - error: creationHookError, - createFeed: async (url: string, token: string) => { - try { - return await mockCreateFeed(url, token); - } catch (error) { - creationHookError = error as FeedCreationError; - throw error; - } - }, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - })); - }); - - const creationFailure = { - kind: 'server' as const, - code: 'INTERNAL_SERVER_ERROR' as const, - retryable: true, - nextAction: 'retry' as const, - retryAction: 'primary' as const, - message: 'Access denied', - }; - - async function renderCreateWithCreationError() { - mockUseApiMetadata.mockReturnValue({ - metadata: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: false, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - isLoading: false, - error: undefined, - }); - mockCreateFeed.mockRejectedValue(creationFailure); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await waitFor(() => { - expect(screen.getByText(COPY.createFailedRetryTitle)).toBeInTheDocument(); - }); - } - - it('renders the radical-simple create flow', () => { - render(); - - expect(screen.getByLabelText('html2rss')).toBeInTheDocument(); - expect(screen.getByRole('link', { name: 'html2rss' }).getAttribute('href')).toMatch(/#\/create$/); - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - expect(screen.queryByRole('combobox')).not.toBeInTheDocument(); - expect(screen.getByLabelText(COPY.utilities)).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.bookmarkletTitle })).toBeInTheDocument(); - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); - }); - - it('keeps the URL field permissive enough for hostname-only input', () => { - render(); - - const urlInput = screen.getByLabelText(COPY.urlLabel); - - expect(urlInput).toHaveAttribute('type', 'text'); - expect(urlInput).toHaveAttribute('inputmode', 'url'); - expect(urlInput).toHaveAttribute('autocapitalize', 'off'); - }); - - it('autofocuses the URL field', async () => { - render(); - - await waitFor(() => { - expect(document.activeElement).toBe(screen.getByLabelText(COPY.urlLabel)); - }); - }); - - it('submits create requests without exposing strategy selection', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - }); - }); - - it('submits create requests when Enter is pressed on the URL field', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.keyDown(screen.getByLabelText(COPY.urlLabel), { key: 'Enter' }); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - }); - }); - - it('does not submit extra create requests when Enter key-repeats', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - const urlField = screen.getByLabelText(COPY.urlLabel); - fireEvent.keyDown(urlField, { key: 'Enter' }); - fireEvent.keyDown(urlField, { key: 'Enter', repeat: true }); - fireEvent.keyDown(urlField, { key: 'Enter', repeat: true }); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledTimes(1); - }); - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - }); - - it('auto-submits a prefilled url without persisting strategy state', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - history.replaceState({}, '', 'http://localhost:3000/?url=https%3A%2F%2Fexample.com%2Farticles'); - - render(); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - expect(location.hash).toBe('#/result/generated-token'); - }); - }); - - it('recovers unmatched result deep links onto remounted create without url prefill', async () => { - history.replaceState({}, '', 'http://localhost:3000/#/result/generated-token'); - - render(); - - await waitFor(() => { - expect(location.hash).toBe('#/create'); - }); - expect(location.hash).not.toContain('url='); - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); - expect(screen.queryByRole('heading', { name: 'Saved result unavailable' })).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - }); - - it('remounts create from BrandLockup and clears creation chrome', async () => { - await renderCreateWithCreationError(); - const createCalls = mockCreateFeed.mock.calls.length; - - fireEvent.click(screen.getByRole('link', { name: 'html2rss' })); - - await expectCreateRemountedWithoutErrorChrome(); - expect(mockCreateFeed).toHaveBeenCalledTimes(createCalls); - }); - - it('remounts create from a #/create visit and clears creation chrome', async () => { - await renderCreateWithCreationError(); - const createCalls = mockCreateFeed.mock.calls.length; - - dispatchEvent(new HashChangeEvent('hashchange')); - - await expectCreateRemountedWithoutErrorChrome(); - expect(location.hash).toBe('#/create'); - expect(mockCreateFeed).toHaveBeenCalledTimes(createCalls); - }); - - it('remounts create from a hashbang entry and clears creation chrome', async () => { - await renderCreateWithCreationError(); - const createCalls = mockCreateFeed.mock.calls.length; - - location.hash = '#!/create'; - - await waitFor(() => { - expect(location.hash).toBe('#/create'); - }); - await expectCreateRemountedWithoutErrorChrome(); - expect(mockCreateFeed).toHaveBeenCalledTimes(createCalls); - }); - - it('shows inline token prompt when submitting without a token', async () => { - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - expect(screen.getByRole('heading', { name: COPY.tokenTitle })).toBeInTheDocument(); - expect(location.hash).toMatch(/^#\/token/); - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'token_prompt'); - expect(document.querySelector('dialog')).toHaveAttribute('open'); - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - expect(screen.getByLabelText(COPY.urlLabel).closest('[inert]')).not.toBeNull(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - expect(screen.queryByRole('combobox')).not.toBeInTheDocument(); - expect(screen.getByLabelText(COPY.utilities)).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.dockerSetup })).toBeInTheDocument(); - expect(screen.getByText(COPY.tokenHint)).toBeInTheDocument(); - expect(screen.queryByText('Paste an access token to keep going.')).not.toBeInTheDocument(); - await waitFor(() => { - expect(document.activeElement).toBe(document.querySelector('#access-token')); - }); - expect(mockCreateFeed).not.toHaveBeenCalled(); - }); - - const faoStarter = { - id: 'fao.org/newsroom', - path: '/fao.org/newsroom.rss', - title: 'FAO Newsroom', - description: 'News and media from the Food and Agriculture Organization.', - channelUrl: 'https://www.fao.org/newsroom', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }; - - it('shows lean included-feed starters on empty create when creation is enabled', async () => { - mockUseCatalogEntries.mockReturnValue(catalogHook([faoStarter], ['fao.org/newsroom'])); - mockUseAccessToken.mockReturnValue({ - token: 'session-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - await waitFor(() => { - expect(screen.getByRole('link', { name: 'FAO Newsroom' })).toHaveAttribute( - 'href', - '/fao.org/newsroom.rss' - ); - }); - const starters = screen.getByRole('status'); - expect(starters.querySelector('.ui-eyebrow')?.textContent).toBe(COPY.feedDirectory); - expect(screen.queryByText(COPY.feedDirectoryIntro)).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.feedDirectoryLearnMore)).not.toBeInTheDocument(); - expect(document.querySelector('.notice')).toBeNull(); - expect(screen.getByRole('list', { name: COPY.feedDirectory })).toBeInTheDocument(); - const directoryLinks = screen.getAllByRole('link', { name: COPY.browseFeedDirectory(1) }); - expect(directoryLinks).toHaveLength(2); - for (const link of directoryLinks) { - expect(link).toHaveAttribute( - 'href', - 'https://html2rss.github.io/feed-directory/#!url=http%3A%2F%2Flocalhost%3A3000%2F' - ); - expect(link).toHaveClass('utility-link'); - } - await waitFor(() => { - expect(document.activeElement).toBe(screen.getByLabelText(COPY.urlLabel)); - }); - }); - - it('hides included-feed starters when the URL field is non-empty', async () => { - mockUseCatalogEntries.mockReturnValue(catalogHook([faoStarter], ['fao.org/newsroom'])); - mockUseAccessToken.mockReturnValue({ - token: 'session-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - await waitFor(() => { - expect(screen.getByRole('link', { name: 'FAO Newsroom' })).toBeInTheDocument(); - }); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'example.com/articles' }, - }); - - expect(screen.queryByRole('link', { name: 'FAO Newsroom' })).not.toBeInTheDocument(); - expect(screen.queryByRole('list', { name: COPY.feedDirectory })).not.toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.browseFeedDirectory(1) })).toHaveAttribute( - 'href', - 'https://html2rss.github.io/feed-directory/#!url=http%3A%2F%2Flocalhost%3A3000%2F' - ); - }); - - it('hides included-feed starters when catalog find has hits', async () => { - mockUseCatalogEntries.mockReturnValue( - catalogHook([ - faoStarter, - { - id: 'anthropic.com/news', - path: '/anthropic.com/news.rss', - title: 'Anthropic — News', - description: 'Product and research announcements from Anthropic.', - channelUrl: 'https://www.anthropic.com/news', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }, - ]) - ); - mockUseAccessToken.mockReturnValue({ - token: 'session-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - await waitFor(() => { - expect(screen.getByRole('link', { name: 'FAO Newsroom' })).toBeInTheDocument(); - }); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'www.anthropic.com/news' }, - }); - - await waitFor(() => { - expect(screen.getByRole('option', { name: 'Anthropic — News' })).toBeInTheDocument(); - }); - expect(screen.queryByRole('link', { name: 'FAO Newsroom' })).not.toBeInTheDocument(); - expect(screen.queryByRole('list', { name: COPY.feedDirectory })).not.toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.browseFeedDirectory(2) })).toBeInTheDocument(); - }); - - it('promotes included feeds when feed creation is disabled', async () => { - mockUseCatalogEntries.mockReturnValue(catalogHook([faoStarter], ['fao.org/newsroom'])); - - mockUseApiMetadata.mockReturnValue({ - metadata: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: false, - access_token_required: false, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - isLoading: false, - error: undefined, - }); - - render(); - - await waitFor(() => { - expect(document.querySelector('.notice__title')?.textContent).toBe(COPY.feedDirectory); - }); - expect(screen.getByRole('link', { name: 'FAO Newsroom' })).toHaveAttribute( - 'href', - '/fao.org/newsroom.rss' - ); - expect(screen.getByText(COPY.creationDisabled)).toBeInTheDocument(); - expect(screen.getByText(COPY.feedDirectoryIntro)).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.feedDirectoryLearnMore })).toHaveAttribute( - 'href', - 'https://html2rss.github.io/web-application/guides/use-the-feed-directory/' - ); - expect(document.querySelector('.notice')).not.toBeNull(); - }); - - it('suggests included feeds when the URL matches the catalog', async () => { - mockUseCatalogEntries.mockReturnValue( - catalogHook([ - { - id: 'anthropic.com/news', - path: '/anthropic.com/news.rss', - title: 'Anthropic — News', - description: 'Product and research announcements from Anthropic.', - channelUrl: 'https://www.anthropic.com/news', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }, - ]) - ); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'www.anthropic.com/news' }, - }); - - await waitFor(() => { - expect(screen.getByRole('option', { name: 'Anthropic — News' })).toHaveAttribute( - 'href', - '/anthropic.com/news.rss' - ); - }); - expect(screen.getByRole('status')).toHaveTextContent(COPY.catalogFindHint); - }); - - it('lists multiple catalog find hits for a text query including defaults href', async () => { - mockUseCatalogEntries.mockReturnValue( - catalogHook([ - { - id: 'bbc.com/mundo', - path: '/bbc.com/mundo.rss', - title: 'BBC — Mundo', - description: 'Spanish-language news from BBC Mundo.', - channelUrl: 'https://www.bbc.com/mundo', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }, - { - id: 'bbc.co.uk/available_episodes', - path: '/bbc.co.uk/available_episodes.rss', - title: 'BBC Sounds — Programme episodes', - description: 'Available episodes for a BBC programme on Sounds.', - channelUrl: 'https://www.bbc.co.uk/programmes/%s/episodes/player', - parameterDefaults: { id: 'b006wkfp' }, - lastResult: { state: 'unknown' }, - }, - ]) - ); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'bbc' }, - }); - - await waitFor(() => { - expect(screen.getByRole('option', { name: 'BBC — Mundo' })).toHaveAttribute( - 'href', - '/bbc.com/mundo.rss' - ); - expect(screen.getByRole('option', { name: 'BBC Sounds — Programme episodes' })).toHaveAttribute( - 'href', - '/bbc.co.uk/available_episodes.rss?id=b006wkfp' - ); - }); - - expect(screen.getByRole('option', { name: 'BBC — Mundo' })).toHaveClass('catalog-hit'); - expect(screen.getByRole('option', { name: 'BBC — Mundo' })).not.toHaveClass('ui-card'); - expect(screen.getByRole('listbox', { name: COPY.catalogFindHitsLabel })).toBeInTheDocument(); - }); - - it('opens the active catalog hit with ArrowDown then Enter without submitting Create', async () => { - const assignSpy = vi.fn(); - vi.stubGlobal('location', { ...location, assign: assignSpy }); - - mockUseCatalogEntries.mockReturnValue( - catalogHook([ - { - id: 'bbc.com/mundo', - path: '/bbc.com/mundo.rss', - title: 'BBC — Mundo', - description: 'Spanish-language news from BBC Mundo.', - channelUrl: 'https://www.bbc.com/mundo', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }, - { - id: 'bbc.co.uk/available_episodes', - path: '/bbc.co.uk/available_episodes.rss', - title: 'BBC Sounds — Programme episodes', - description: 'Available episodes for a BBC programme on Sounds.', - channelUrl: 'https://www.bbc.co.uk/programmes/%s/episodes/player', - parameterDefaults: { id: 'b006wkfp' }, - lastResult: { state: 'unknown' }, - }, - ]) - ); - - render(); - - const urlField = screen.getByLabelText(COPY.urlLabel); - fireEvent.input(urlField, { target: { value: 'bbc' } }); - - await waitFor(() => { - expect(screen.getByRole('listbox', { name: COPY.catalogFindHitsLabel })).toBeInTheDocument(); - }); - - fireEvent.keyDown(urlField, { key: 'ArrowDown' }); - await waitFor(() => { - expect(urlField).toHaveAttribute('aria-activedescendant', 'catalog-find-hits-option-0'); - }); - - fireEvent.keyDown(urlField, { key: 'Enter' }); - - expect(assignSpy).toHaveBeenCalledWith('/bbc.co.uk/available_episodes.rss?id=b006wkfp'); - expect(mockCreateFeed).not.toHaveBeenCalled(); - - vi.unstubAllGlobals(); - }); - - it('submits Create with Enter when no catalog hit is active', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - mockUseCatalogEntries.mockReturnValue( - catalogHook([ - { - id: 'bbc.com/mundo', - path: '/bbc.com/mundo.rss', - title: 'BBC — Mundo', - description: 'Spanish-language news from BBC Mundo.', - channelUrl: 'https://www.bbc.com/mundo', - parameterDefaults: {}, - lastResult: { state: 'unknown' }, - }, - ]) - ); - - render(); - - const urlField = screen.getByLabelText(COPY.urlLabel); - fireEvent.input(urlField, { target: { value: 'https://www.bbc.com/mundo' } }); - - await waitFor(() => { - expect(screen.getByRole('listbox', { name: COPY.catalogFindHitsLabel })).toBeInTheDocument(); - }); - - fireEvent.keyDown(urlField, { key: 'Enter' }); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://www.bbc.com/mundo', 'saved-token'); - }); - }); - - it('renders the result panel when a feed is available', async () => { - history.replaceState({}, '', 'http://localhost:3000/#/result/example-token'); - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: { - feed: { - id: 'feed-123', - name: 'Example Feed', - url: 'https://example.com/articles', - feed_token: 'example-token', - public_url: '/api/v1/feeds/example-token', - json_public_url: '/api/v1/feeds/example-token.json', - }, - preview: { - status: 'preview_failed' as const, - items: [], - isLoading: false, - }, - warnings: [ - { - code: 'preview_unavailable', - message: 'Preview unavailable right now.', - retryable: false, - nextAction: 'none', - }, - ], - }, - error: undefined, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - - render(); - - expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'result'); - expect(screen.getByRole('button', { name: COPY.createAnother })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.openFeed })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.bookmarkletTitle })).toBeInTheDocument(); - expect(screen.getByText('Example Feed')).toBeInTheDocument(); - expect(screen.getByText(COPY.feedReady)).toBeInTheDocument(); - expect(screen.getByText(COPY.previewUnavailable)).toBeInTheDocument(); - - fireEvent.click(screen.getByRole('button', { name: COPY.createAnother })); - return waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - }); - - it('surfaces creation errors to the user', () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'server', - code: 'INTERNAL_SERVER_ERROR', - retryable: true, - nextAction: 'retry', - retryAction: 'primary', - message: 'Access denied', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - - render(); - - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'error'); - expect(screen.getByText(COPY.createFailedRetryTitle)).toBeInTheDocument(); - expect(screen.getByText('Access denied')).toBeInTheDocument(); - }); - - it('presents classified create errors with the wire sentence', () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'input', - code: 'BLOCKED_SURFACE', - retryable: false, - nextAction: 'correct_input', - retryAction: 'none', - message: 'This site blocked automated access. Try another URL or site.', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - - render(); - - expect(screen.getByText(COPY.createFailedTitle)).toBeInTheDocument(); - expect( - screen.getByText('This site blocked automated access. Try another URL or site.') - ).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: COPY.tryAgain })).not.toBeInTheDocument(); - }); - - it('shows instance metadata failure as a banner without create-error chrome', () => { - mockUseApiMetadata.mockReturnValue({ - metadata: undefined, - isLoading: false, - error: 'Instance unavailable.', - }); - - render(); - - expect(screen.getByText(COPY.instanceMetadataUnavailable)).toBeInTheDocument(); - expect(screen.getByText(COPY.instanceUnavailable)).toBeInTheDocument(); - expect(screen.queryByText(COPY.accessTokenUnavailable)).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - expect(document.querySelector('.form-shell')).toHaveAttribute('data-state', 'create'); - }); - - it('shows access token load failure with a token title, not metadata heading', () => { - mockUseAccessToken.mockReturnValue({ - token: undefined, - hasToken: false, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: COPY.unableToLoadToken, - }); - - render(); - - expect(screen.getByText(COPY.accessTokenUnavailable)).toBeInTheDocument(); - expect(screen.getByText(COPY.unableToLoadToken)).toBeInTheDocument(); - expect(screen.queryByText(COPY.instanceMetadataUnavailable)).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - }); - - it('shows title-only creating notice without preview copy', () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: true, - result: undefined, - error: undefined, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - - render(); - - expect(screen.getByText(COPY.creating)).toBeInTheDocument(); - expect(screen.queryByText(COPY.previewChecking)).not.toBeInTheDocument(); - }); - - it('keeps the token dialog open while creating', () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: true, - result: undefined, - error: undefined, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - history.replaceState({}, '', 'http://localhost:3000/#/token?url=https%3A%2F%2Fexample.com%2Farticles'); - - render(); - - expect(document.querySelector('dialog')).toHaveAttribute('open'); - expect(screen.getByRole('button', { name: COPY.creating })).toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - }); - - it('clears stored token from instance info', () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.click(screen.getByRole('button', { name: COPY.logout })); - - expect(mockClearToken).toHaveBeenCalled(); - }); - - it('keeps the Docker Hub link before token clear when a token is saved', () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - const utilityItems = [ - ...screen - .getByLabelText(COPY.utilities) - .querySelectorAll(':scope .utility-strip__items > a, :scope .utility-strip__items > button'), - ].map((element) => element.textContent); - - expect(utilityItems).toEqual([ - COPY.browseFeedDirectory(), - COPY.bookmarkletTitle, - COPY.logout, - COPY.dockerInstall, - COPY.openapiSpec, - COPY.sourceCode, - ]); - }); - - it('saves access token and resumes feed creation from the inline prompt', async () => { - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - const accessTokenInput = document.querySelector('#access-token') as HTMLInputElement; - fireEvent.input(accessTokenInput, { target: { value: 'token-123' } }); - fireEvent.click(screen.getByRole('button', { name: COPY.saveAndContinue })); - - await waitFor(() => { - expect(mockSaveToken).toHaveBeenCalledWith('token-123'); - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'token-123'); - }); - }); - - it('reopens the token prompt when a saved token is rejected', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - mockCreateFeed.mockRejectedValueOnce( - Object.assign(new Error('Unauthorized'), { - code: 'UNAUTHORIZED', - status: 401, - kind: 'auth', - }) - ); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await waitFor(() => { - expect(screen.getByRole('heading', { name: COPY.tokenTitle })).toBeInTheDocument(); - expect(screen.getByText(COPY.tokenRejected)).toBeInTheDocument(); - expect(mockClearToken).toHaveBeenCalled(); - expect(mockClearCreationError).toHaveBeenCalled(); - }); - }); - - it('clears stale creation error when backing out of the token gate', async () => { - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - mockCreateFeed.mockRejectedValueOnce( - Object.assign(new Error('Unauthorized'), { - code: 'UNAUTHORIZED', - status: 401, - kind: 'auth', - }) - ); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - await screen.findByText(COPY.tokenRejected); - fireEvent.click(screen.getByRole('button', { name: COPY.back })); - - await waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); - expect(screen.queryByText('Unauthorized')).not.toBeInTheDocument(); - }); - - it('submits the token prompt with Enter', async () => { - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - const accessTokenInput = document.querySelector('#access-token') as HTMLInputElement; - fireEvent.input(accessTokenInput, { target: { value: 'token-123' } }); - fireEvent.keyDown(accessTokenInput, { key: 'Enter' }); - - await waitFor(() => { - expect(mockSaveToken).toHaveBeenCalledWith('token-123'); - }); - }); - - it('builds a bookmarklet that returns to the root app entry', () => { - history.replaceState({}, '', 'http://localhost:3000/#/create'); - render(); - - const bookmarklet = screen.getByRole('link', { name: COPY.bookmarkletTitle }); - expect(bookmarklet.getAttribute('href')).toContain('/#/create?url='); - expect(bookmarklet.getAttribute('href')).not.toContain('%27+encodeURIComponent'); - }); - - it('opens token entry immediately for bookmarklet urls when no token is saved', async () => { - history.replaceState({}, '', 'http://localhost:3000/?url=example.com%2Farticles'); - - render(); - - await screen.findByRole('heading', { name: COPY.tokenTitle }); - expect(location.hash).toMatch(/^#\/token/); - expect(document.querySelector('dialog')).toHaveAttribute('open'); - expect(screen.getByLabelText(COPY.urlLabel)).toBeInTheDocument(); - await waitFor(() => { - expect(screen.getByLabelText(COPY.urlLabel)).toHaveValue('https://example.com/articles'); - }); - expect(mockCreateFeed).not.toHaveBeenCalled(); - }); - - it('prefills the URL field when landing with a bookmarklet hash route', async () => { - history.replaceState({}, '', 'http://localhost:3000/#/create?url=https%3A%2F%2Fexample.com%2Farticles'); - - render(); - - await waitFor(() => { - expect(screen.getByLabelText(COPY.urlLabel)).toHaveValue('https://example.com/articles'); - }); - }); - - it('prefers bookmarklet route prefill over a saved draft url', async () => { - localStorage.setItem( - 'html2rss_feed_draft_state', - JSON.stringify({ url: 'https://old.example.com/page' }) - ); - history.replaceState({}, '', 'http://localhost:3000/#/create?url=https%3A%2F%2Fexample.com%2Farticles'); - - render(); - - await waitFor(() => { - expect(screen.getByLabelText(COPY.urlLabel)).toHaveValue('https://example.com/articles'); - }); - }); - - it('shows generic retry action for alternate retry metadata and reruns create', async () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'server', - code: 'INTERNAL_SERVER_ERROR', - retryable: true, - nextAction: 'retry', - retryAction: 'alternate', - message: 'Browserless failed.', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.tryAgain })); - - expect(screen.queryByRole('button', { name: /Retry with .*/ })).not.toBeInTheDocument(); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - }); - }); - - it('shows Try again for primary retry metadata and reruns the create flow', async () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'server', - code: 'INTERNAL_SERVER_ERROR', - retryable: true, - nextAction: 'retry', - retryAction: 'primary', - message: 'Browserless failed.', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.tryAgain })); - - await waitFor(() => { - expect(mockCreateFeed).toHaveBeenCalledWith('https://example.com/articles', 'saved-token'); - }); - }); - - it('does not treat non-token forbidden failures as token rejection', async () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'server', - code: 'FORBIDDEN', - retryable: false, - nextAction: 'none', - retryAction: 'none', - message: 'URL not allowed for this account', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - await screen.findByText('URL not allowed for this account'); - expect(mockClearToken).not.toHaveBeenCalled(); - expect(screen.queryByRole('heading', { name: COPY.tokenTitle })).not.toBeInTheDocument(); - expect(screen.queryByText(COPY.tokenRejected)).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: COPY.tryAgain })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /Retry with .*/ })).not.toBeInTheDocument(); - }); - - it('keeps extraction-empty failures generic and input-corrective', async () => { - mockUseFeedCreation.mockReturnValue({ - isCreating: false, - result: undefined, - error: { - kind: 'input', - code: 'NO_FEED_ITEMS_EXTRACTED', - retryable: false, - nextAction: 'correct_input', - retryAction: 'none', - message: 'Could not extract feed items. Try a more specific listing URL or explicit selectors.', - }, - createFeed: mockCreateFeed, - clearError: mockClearCreationError, - clearResult: mockClearResult, - retryPreviewFetch: mockRetryPreviewFetch, - }); - mockUseAccessToken.mockReturnValue({ - token: 'saved-token', - hasToken: true, - saveToken: mockSaveToken, - clearToken: mockClearToken, - isLoading: false, - error: undefined, - }); - - render(); - - await screen.findByText( - 'Could not extract feed items. Try a more specific listing URL or explicit selectors.' - ); - expect(screen.getByText(COPY.createFailedTitle)).toBeInTheDocument(); - expect(screen.queryByText(COPY.createFailedRetryTitle)).not.toBeInTheDocument(); - expect(screen.queryByRole('heading', { name: COPY.tokenTitle })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: COPY.tryAgain })).not.toBeInTheDocument(); - expect(mockClearToken).not.toHaveBeenCalled(); - }); - - it('returns non-auth token save failures onto create with notice', async () => { - mockCreateFeed.mockRejectedValueOnce({ - kind: 'server', - code: 'INTERNAL_SERVER_ERROR', - retryable: true, - nextAction: 'retry', - retryAction: 'primary', - message: 'Upstream failed', - }); - - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - const accessTokenInput = document.querySelector('#access-token') as HTMLInputElement; - fireEvent.input(accessTokenInput, { target: { value: 'token-123' } }); - fireEvent.click(screen.getByRole('button', { name: COPY.saveAndContinue })); - - await waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - expect(document.querySelector('dialog')).toBeNull(); - expect(screen.getByText(COPY.createFailedRetryTitle)).toBeInTheDocument(); - expect(screen.getByText('Upstream failed')).toBeInTheDocument(); - }); - - it('cancels the token dialog with Back, Escape, and backdrop', async () => { - render(); - - fireEvent.input(screen.getByLabelText(COPY.urlLabel), { - target: { value: 'https://example.com/articles' }, - }); - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - - expect(document.querySelector('dialog')).toHaveAttribute('open'); - const accessTokenInput = document.querySelector('#access-token') as HTMLInputElement; - fireEvent.input(accessTokenInput, { target: { value: 'secret-draft' } }); - fireEvent.click(screen.getByRole('button', { name: COPY.back })); - - await waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - expect(document.querySelector('dialog')).toBeNull(); - - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - const dialog = document.querySelector('dialog'); - expect(dialog).toHaveAttribute('open'); - expect((document.querySelector('#access-token') as HTMLInputElement).value).toBe(''); - fireEvent(dialog as HTMLDialogElement, new Event('cancel', { bubbles: true })); - - await waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - - fireEvent.click(screen.getByRole('button', { name: COPY.createFeed })); - const openDialog = document.querySelector('dialog') as HTMLDialogElement; - expect(openDialog).toHaveAttribute('open'); - fireEvent.click(openDialog); - - await waitFor(() => { - expect(location.hash).toMatch(/^#\/create/); - }); - expect(document.querySelector('dialog')).toBeNull(); - }); - - it('shows the utility links in a user-focused order', () => { - history.replaceState({}, '', 'http://localhost:3000/#/create'); - render(); - - const utilityLinks = [ - ...screen.getByLabelText(COPY.utilities).querySelectorAll(':scope .utility-strip__items > a'), - ].map((link) => link.textContent); - expect(utilityLinks).toEqual([ - COPY.browseFeedDirectory(), - COPY.bookmarkletTitle, - COPY.dockerInstall, - COPY.openapiSpec, - COPY.sourceCode, - ]); - - expect(screen.getByRole('link', { name: COPY.openapiSpec })).toHaveAttribute( - 'href', - 'https://example.test/openapi.yaml' - ); - expect(screen.getByRole('link', { name: COPY.browseFeedDirectory() })).toHaveAttribute( - 'href', - 'https://html2rss.github.io/feed-directory/#!url=http%3A%2F%2Flocalhost%3A3000%2F' - ); - expect(screen.getByRole('link', { name: COPY.dockerInstall })).toHaveAttribute( - 'href', - 'https://hub.docker.com/r/html2rss/web' - ); - }); - - it('keeps OpenAPI link on the frontend origin during local development', () => { - mockUseApiMetadata.mockReturnValue({ - metadata: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'http://127.0.0.1:4000/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: true, - }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - }, - isLoading: false, - error: undefined, - }); - - history.replaceState({}, '', 'http://localhost:3000/#/create'); - render(); - - expect(screen.getByRole('link', { name: COPY.openapiSpec })).toHaveAttribute( - 'href', - 'http://localhost:3000/openapi.yaml' - ); - }); - - it('shows footer utilities on result routes', async () => { - history.replaceState({}, '', 'http://localhost:3000/#/result/generated-token'); - render(); - - await waitFor(() => { - expect(screen.getByLabelText(COPY.utilities)).toBeInTheDocument(); - }); - }); -}); diff --git a/frontend/src/__tests__/ConfigStudio.test.tsx b/frontend/src/__tests__/ConfigStudio.test.tsx new file mode 100644 index 000000000..3b2a49812 --- /dev/null +++ b/frontend/src/__tests__/ConfigStudio.test.tsx @@ -0,0 +1,308 @@ +import { describe, it, expect, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor, within } from '@testing-library/preact'; +import { http, HttpResponse } from 'msw'; +import type { StudioSelectors } from '../studio/selectorDraft'; +import { server } from './mocks/server'; +import { + buildStructuredErrorResponse, + DEFAULT_SUGGEST_BUCKETS, + selectorsFromValidateRequest, + pageUrlFrom, + suggestSuccessBody, + validateSuccessBody, + validateYamlForSelectors, + studioPreviewSuccessBody, +} from './mocks/apiFixtures'; +import { ConfigStudio } from '../studio'; +import { COPY } from '../journey/copy'; + +vi.mock('../studio/useStudio', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useStudio: (options: Parameters[0]) => + actual.useStudio({ ...options, validateDelayMs: 0, previewSettleMs: 0 }), + }; +}); + +const pageUrl = 'https://example.com/articles'; +const decisionNotice = 'We could not extract feed items from this page yet.'; +const imagePreview = studioPreviewSuccessBody([ + { + title: 'With image', + url: 'https://example.com/sample', + published_at: '2024-01-02', + image: 'https://cdn.example/a.jpg', + }, + { title: 'No image', url: 'https://example.com/plain' }, +]); + +function echoValidate() { + server.use( + http.post('/api/v1/feeds/validate', async ({ request }) => { + const body = await request.json(); + const selectors = selectorsFromValidateRequest(body); + const yaml = validateYamlForSelectors(selectors, pageUrlFrom(body)); + return HttpResponse.json(validateSuccessBody(selectors, yaml)); + }) + ); +} + +function stubSuggest(body: unknown = suggestSuccessBody(DEFAULT_SUGGEST_BUCKETS)) { + server.use(http.post('/api/v1/feeds/suggest_selectors', () => HttpResponse.json(body))); +} + +function stubPreview(body: unknown = studioPreviewSuccessBody()) { + server.use(http.post('/api/v1/feeds/preview', () => HttpResponse.json(body))); +} + +function renderStudio( + onGenerate = vi.fn<(selectors: StudioSelectors) => Promise>(), + extras: Partial<{ + mode: 'ready' | 'unresolved'; + directoryHandoff: boolean; + decisionNotice: string; + token: string; + }> = {} +) { + render( + + ); + return { onGenerate }; +} + +function itemsChip(name = COPY.addSelector('Sample headline')) { + return within(screen.getByRole('group', { name: COPY.selectorChoices })).getByRole('button', { name }); +} + +async function waitForItemsChip(name = COPY.addSelector('Sample headline')) { + await waitFor(() => expect(itemsChip(name)).toBeInTheDocument()); + return itemsChip(name); +} + +describe('ConfigStudio', () => { + it('keeps Save/meadow visible while suggestions pending and densifies fields', async () => { + echoValidate(); + stubPreview(imagePreview); + const pending = Promise.withResolvers(); + server.use(http.post('/api/v1/feeds/suggest_selectors', () => pending.promise)); + renderStudio(); + + expect(screen.getByText(COPY.selectorSuggestionsLoading)).toBeInTheDocument(); + expect(screen.getByLabelText(COPY.itemsSelector)).not.toHaveClass('input--lg'); + expect(screen.queryByText(COPY.previewFetchedCaption)).not.toBeInTheDocument(); + const save = screen.getByRole('button', { name: COPY.saveAndGenerate }); + expect( + save.compareDocumentPosition(screen.getByLabelText(COPY.previewRegion)) & + Node.DOCUMENT_POSITION_FOLLOWING + ).not.toBe(0); + + pending.resolve( + HttpResponse.json( + suggestSuccessBody({ + items: [{ selector: 'article', enhance: false, sample: 'Sample headline' }], + title: [{ selector: 'h2', sample: 'Headline' }], + }) + ) + ); + await waitFor(() => + expect(screen.getByLabelText(`${COPY.fieldTitle} ${COPY.selector}`)).toBeInTheDocument() + ); + expect(screen.getAllByText(COPY.fieldTitle)).toHaveLength(1); + expect(document.querySelector('details')).toBeNull(); + expect(document.querySelector('.studio-enhance__label')).not.toHaveClass('ui-card'); + + fireEvent.click(await waitForItemsChip()); + await waitFor(() => expect(screen.getByText('With image')).toBeInTheDocument()); + expect(screen.getByText(COPY.previewFetchedCaption)).toBeInTheDocument(); + expect(screen.getByLabelText(COPY.previewRegion).querySelector('.ui-item-list--grid')).toBeTruthy(); + }); + + it('activates dormant fields, keeps empty suggestions neutral, and toggles chips without enhance', async () => { + echoValidate(); + stubSuggest(suggestSuccessBody({})); + const { unmount } = render( + + ); + await waitFor(() => expect(screen.getByText(COPY.selectorSuggestionsEmpty)).toBeInTheDocument()); + expect(screen.queryByRole('group', { name: COPY.selectorChoices })).not.toBeInTheDocument(); + + unmount(); + stubSuggest(suggestSuccessBody({ items: [{ selector: 'article', enhance: true, sample: 'Story A' }] })); + renderStudio(); + await waitForItemsChip(COPY.addSelector('Story A')); + fireEvent.click(screen.getByRole('button', { name: COPY.addOptionalField(COPY.fieldTitle) })); + expect(screen.getByLabelText(`${COPY.fieldTitle} ${COPY.selector}`)).toHaveFocus(); + + fireEvent.click(screen.getByLabelText(COPY.enhanceItems)); + fireEvent.click(itemsChip(COPY.addSelector('Story A'))); + expect(screen.getByLabelText(COPY.itemsSelector)).toHaveValue('a[href], article'); + expect(screen.getByLabelText(COPY.enhanceItems)).toBeChecked(); + fireEvent.click(itemsChip(COPY.removeSelector('Story A'))); + expect(screen.getByLabelText(COPY.itemsSelector)).toHaveValue('a[href]'); + expect(screen.getByLabelText(COPY.enhanceItems)).toBeChecked(); + }); + + it('keeps editing when suggestions fail and supports retry', async () => { + echoValidate(); + let attempts = 0; + server.use( + http.post('/api/v1/feeds/suggest_selectors', () => { + attempts += 1; + if (attempts === 1) { + return HttpResponse.json( + buildStructuredErrorResponse({ + code: 'STUDIO_DISABLED', + message: 'Selector suggestions are unavailable', + kind: 'input', + retryable: false, + next_action: 'none', + retry_action: 'none', + }), + { status: 403 } + ); + } + return HttpResponse.json( + suggestSuccessBody({ items: [{ selector: 'article.retry', sample: 'Sample headline' }] }) + ); + }) + ); + renderStudio(); + expect(await screen.findByText('Selector suggestions are unavailable')).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: COPY.retrySelectorSuggestions })); + await waitForItemsChip(); + expect(attempts).toBe(2); + }); + + it('shows field validation inline without issue-count panels', async () => { + server.use( + http.post('/api/v1/feeds/validate', () => + HttpResponse.json({ + success: true, + data: { + report: { + success: false, + issues: [ + { path: ['selectors', 'title', 'selector'], code: 'missing_key', message: 'is missing' }, + { path: ['selectors', 'url', 'selector'], code: 'constraint', message: 'is invalid' }, + ], + }, + selectors: { items: { selector: 'article', enhance: false } }, + yaml: 'selectors: {}', + }, + }) + ) + ); + renderStudio(); + await waitFor(() => expect(screen.getByText('is missing')).toBeInTheDocument()); + expect(screen.getByText('is invalid')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: /issue/i })).not.toBeInTheDocument(); + }); + + it('gates sample images on token and offers check again on preview failure', async () => { + echoValidate(); + stubPreview(imagePreview); + let view = render(); + fireEvent.click(await waitForItemsChip()); + await waitFor(() => expect(screen.getByText('With image')).toBeInTheDocument()); + expect(document.querySelectorAll('img.studio-sample-image')).toHaveLength(1); + + view.unmount(); + stubPreview(imagePreview); + view = render(); + fireEvent.click(await waitForItemsChip()); + await waitFor(() => expect(screen.getByText('With image')).toBeInTheDocument()); + expect(document.querySelector('img')).toBeNull(); + + view.unmount(); + echoValidate(); + stubSuggest(); + server.use(http.post('/api/v1/feeds/preview', () => HttpResponse.error())); + renderStudio(); + fireEvent.click(await waitForItemsChip()); + expect(await screen.findByRole('button', { name: COPY.checkAgain })).toBeInTheDocument(); + }); + + it('copies yaml and activates ready Save after a candidate edit', async () => { + echoValidate(); + const { onGenerate } = renderStudio(vi.fn(), { mode: 'ready', directoryHandoff: true }); + const save = screen.getByRole('button', { name: COPY.saveAndGenerate }); + expect(save).toBeDisabled(); + fireEvent.click(await waitForItemsChip()); + await waitFor(() => expect(save).toBeEnabled()); + expect(save).toHaveClass('btn--primary'); + expect(document.querySelector('.studio-actions')).toContainElement( + screen.getByRole('link', { name: COPY.proposeDirectory }) + ); + + vi.mocked(navigator.clipboard.writeText).mockRejectedValueOnce(new Error('denied')); + fireEvent.click(screen.getByRole('button', { name: COPY.copyYaml })); + await waitFor(() => expect(document.querySelector('[data-clipboard="failed"]')).toBeTruthy()); + fireEvent.click(screen.getByRole('button', { name: COPY.copyYaml })); + await waitFor(() => expect(navigator.clipboard.writeText).toHaveBeenCalled()); + + fireEvent.click(save); + await waitFor(() => expect(onGenerate).toHaveBeenCalled()); + expect(onGenerate.mock.calls[0]?.[0]).toEqual({ + items: { selector: 'a[href], article', enhance: false }, + }); + }); + + it('does not re-echo Decision on save/suggest failure; keeps distinct save Notice', async () => { + echoValidate(); + server.use( + http.post('/api/v1/feeds/suggest_selectors', () => + HttpResponse.json( + buildStructuredErrorResponse({ + code: 'EXTRACTION_EMPTY', + message: decisionNotice, + kind: 'input', + retryable: false, + next_action: 'none', + retry_action: 'none', + }), + { status: 422 } + ) + ) + ); + const echoSave = vi + .fn<(selectors: StudioSelectors) => Promise>() + .mockRejectedValue(new Error(decisionNotice)); + const { unmount } = render( + + ); + expect(await screen.findByText(COPY.selectorSuggestionsFailed)).toBeInTheDocument(); + expect(screen.queryByText(decisionNotice)).not.toBeInTheDocument(); + const save = screen.getByRole('button', { name: COPY.saveAndGenerate }); + await waitFor(() => expect(save).toBeEnabled()); + fireEvent.click(save); + await waitFor(() => expect(echoSave).toHaveBeenCalled()); + expect(screen.queryByText(COPY.createFailedTitle)).not.toBeInTheDocument(); + + unmount(); + echoValidate(); + stubSuggest(); + const distinct = vi + .fn<(selectors: StudioSelectors) => Promise>() + .mockRejectedValue(new Error('Unable to reach the server.')); + renderStudio(distinct, { decisionNotice }); + const distinctSave = screen.getByRole('button', { name: COPY.saveAndGenerate }); + await waitFor(() => expect(distinctSave).toBeEnabled()); + fireEvent.click(distinctSave); + expect(await screen.findByText(COPY.createFailedTitle)).toBeInTheDocument(); + expect(screen.getByText('Unable to reach the server.')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/__tests__/ResultDisplay.test.tsx b/frontend/src/__tests__/ResultDisplay.test.tsx index 810e66924..67f51a427 100644 --- a/frontend/src/__tests__/ResultDisplay.test.tsx +++ b/frontend/src/__tests__/ResultDisplay.test.tsx @@ -5,10 +5,11 @@ import { COPY } from '../journey/copy'; import type { AppViewModel } from '../feed'; describe('ResultDisplay', () => { - const mockOnCreateAnother = vi.fn(); - const mockOnRetryPreview = vi.fn(); - const mockViewModel: Extract = { + const onCreateAnother = vi.fn(); + const onRetryPreview = vi.fn(); + const ready: Extract = { kind: 'result', + phase: 'ready', feed: { id: 'test-id', name: 'Test Feed', @@ -16,8 +17,6 @@ describe('ResultDisplay', () => { feed_token: 'test-feed-token', public_url: 'https://example.com/feed.xml', json_public_url: 'https://example.com/feed.json', - created_at: '2024-01-01T00:00:00Z', - updated_at: '2024-01-01T00:00:00Z', }, preview: { status: 'preview_ready', @@ -38,50 +37,12 @@ describe('ResultDisplay', () => { vi.clearAllMocks(); }); - it('renders ready feed with Copy as primary CTA and demoted open links', async () => { - const viewModelWithMultiplePreviewItems = { - ...mockViewModel, - preview: { - status: 'preview_ready' as const, - items: [ - { - title: 'Item One', - excerpt: 'First preview item with markup.', - url: 'https://example.com/item-one', - publishedLabel: 'Jan 1, 2024', - }, - { - title: 'Item Two', - excerpt: 'Second preview item with markup.', - url: 'https://example.com/item-two', - publishedLabel: 'Jan 2, 2024', - }, - { - title: 'Item Three', - excerpt: 'Third preview item with markup.', - url: 'https://example.com/item-three', - publishedLabel: 'Jan 3, 2024', - }, - { - title: 'Item Four', - excerpt: 'Fourth preview item with markup.', - url: 'https://example.com/item-four', - publishedLabel: 'Jan 4, 2024', - }, - ], - isLoading: false, - }, - }; - + it('renders ready feed with Copy primary, demoted opens, and grid meadow', async () => { render( - + ); - expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'result'); + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'ready'); expect(screen.getByText(COPY.feedReady)).toBeInTheDocument(); expect(screen.getByRole('button', { name: COPY.copyFeedUrl })).toBeInTheDocument(); expect(screen.getByRole('link', { name: COPY.openFeed })).toHaveClass('btn--ghost'); @@ -91,36 +52,53 @@ describe('ResultDisplay', () => { ); await waitFor(() => { expect(screen.getByText('Item One')).toBeInTheDocument(); - expect(screen.getByText('Item Four')).toBeInTheDocument(); - expect(screen.getByText(COPY.previewItemCount(4))).toBeInTheDocument(); + expect(screen.getByText(COPY.previewItemCount(1))).toBeInTheDocument(); + expect(document.querySelector('.ui-item-list--grid')).toBeTruthy(); }); - expect(screen.queryByRole('button', { name: /show all .* items/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: 'Show fewer items' })).not.toBeInTheDocument(); }); - it('keeps Copy and open actions available while preview loads', () => { - render( + it('keeps Copy while preview loads and surfaces preview warnings', () => { + const { rerender } = render( ); - expect(screen.getByText(COPY.feedReady)).toBeInTheDocument(); expect(screen.getByText(COPY.previewChecking)).toBeInTheDocument(); expect(screen.getByRole('button', { name: COPY.copyFeedUrl })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: COPY.openFeed })).toBeInTheDocument(); - }); - it('lets retryable preview failures retry preview only', () => { - render( + rerender( + + ); + expect( + screen.getByText('This site blocked automated access. Try another URL or site.') + ).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: COPY.checkAgain })).not.toBeInTheDocument(); + + rerender( { }, ], }} - onCreateAnother={mockOnCreateAnother} - onRetryPreview={mockOnRetryPreview} + onCreateAnother={onCreateAnother} + onRetryPreview={onRetryPreview} /> ); - - expect(screen.getByText(COPY.feedReady)).toBeInTheDocument(); fireEvent.click(screen.getByRole('button', { name: COPY.checkAgain })); - expect(mockOnRetryPreview).toHaveBeenCalled(); + expect(onRetryPreview).toHaveBeenCalled(); }); - it('calls onCreateAnother and copies feed URL', async () => { + it('copies from the action, ignores Enter on the feed URL, and surfaces clipboard failure', async () => { render( - + ); fireEvent.click(screen.getByRole('button', { name: COPY.createAnother })); - expect(mockOnCreateAnother).toHaveBeenCalled(); + expect(onCreateAnother).toHaveBeenCalled(); + + fireEvent.keyDown(screen.getByLabelText(COPY.feedUrl), { key: 'Enter' }); + expect(navigator.clipboard.writeText).not.toHaveBeenCalled(); fireEvent.click(screen.getByRole('button', { name: COPY.copyFeedUrl })); await waitFor(() => { expect(navigator.clipboard.writeText).toHaveBeenCalledWith('https://example.com/feed.xml'); }); - }); - it('does not copy when Enter is pressed on the feed URL field', () => { - render( - - ); - - const feedUrlField = screen.getByLabelText(COPY.feedUrl); - fireEvent.keyDown(feedUrlField, { key: 'Enter' }); - fireEvent.keyDown(feedUrlField, { key: 'Enter', repeat: true }); - - expect(navigator.clipboard.writeText).not.toHaveBeenCalled(); + vi.mocked(navigator.clipboard.writeText).mockRejectedValueOnce(new Error('denied')); + fireEvent.click(screen.getByRole('button', { name: COPY.copyFeedUrl })); + await waitFor(() => { + expect(document.querySelector('[data-clipboard="failed"]')).toBeTruthy(); + }); + expect(screen.getByLabelText(COPY.feedUrl)).toHaveValue('https://example.com/feed.xml'); }); - it('renders PREVIEW_HTTP warning messages from the loader', () => { - render( + it('mounts authenticated studio for ready results, focuses Copy, and hides controls without studio', async () => { + const { rerender } = render( {}), }} - onCreateAnother={mockOnCreateAnother} - onRetryPreview={mockOnRetryPreview} /> ); - expect( - screen.getByText('This site blocked automated access. Try another URL or site.') - ).toBeInTheDocument(); - expect(screen.queryByRole('button', { name: COPY.checkAgain })).not.toBeInTheDocument(); + const copyFeedUrl = screen.getByRole('button', { name: COPY.copyFeedUrl }); + expect(document.querySelector('details')).toBeNull(); + expect(screen.getByLabelText(COPY.itemsSelector)).toBeInTheDocument(); + expect(document.querySelectorAll('.ui-item-list')).toHaveLength(1); + expect(document.querySelector('.ui-item-list--grid')).toBeTruthy(); + expect(screen.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass('btn--ghost'); + expect(screen.getByRole('link', { name: COPY.proposeDirectory })).toBeInTheDocument(); + await waitFor(() => { + expect(document.activeElement).toBe(copyFeedUrl); + }); + + rerender( + + ); + expect(screen.queryByLabelText(COPY.itemsSelector)).not.toBeInTheDocument(); + expect(screen.getByRole('button', { name: COPY.createAnother })).toBeInTheDocument(); }); - it('renders retryable PREVIEW_HTTP warnings with Check again', () => { + it('renders unresolved workspace with one Decision notice and open studio', () => { + const notice = 'We could not extract feed items from this page yet.'; render( {}), }} - onCreateAnother={mockOnCreateAnother} - onRetryPreview={mockOnRetryPreview} /> ); - expect(screen.getByText('Feed fetching is temporarily unavailable.')).toBeInTheDocument(); - expect(screen.getByRole('button', { name: COPY.checkAgain })).toBeInTheDocument(); + expect(document.querySelector('.result-shell')).toHaveAttribute('data-state', 'unresolved'); + expect(screen.getByText('https://example.com/articles')).toBeInTheDocument(); + expect(screen.getAllByText(notice)).toHaveLength(1); + expect(screen.queryByText(COPY.feedReady)).not.toBeInTheDocument(); + expect(screen.queryByRole('button', { name: COPY.copyFeedUrl })).not.toBeInTheDocument(); + expect(screen.getByLabelText(COPY.itemsSelector)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: COPY.saveAndGenerate })).toHaveClass('btn--primary'); + expect(document.querySelector('.studio-suggestion-status')).toBeTruthy(); }); }); diff --git a/frontend/src/__tests__/apiHttp.test.ts b/frontend/src/__tests__/apiHttp.test.ts new file mode 100644 index 000000000..a88c3c15b --- /dev/null +++ b/frontend/src/__tests__/apiHttp.test.ts @@ -0,0 +1,150 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getApiMetadata } from '../api/generated/sdk.gen'; +import { httpRequestUrl, jsonBaseUrl, unwrap } from '../api/http/client'; +import { requestConfigCatalog } from '../api/http/catalog'; +import { postCreateFeed, postPreviewFeed, postSuggestSelectors, postValidateFeed } from '../api/http/feeds'; +import { requestApiMetadata } from '../api/http/metadata'; +import { createdFeedBody, emptyCatalog, metadataBody } from './mocks/apiFixtures'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function fetchRequest(fetchMock: ReturnType, callIndex = 0): Request { + const input = fetchMock.mock.calls[callIndex]?.[0]; + if (!(input instanceof Request)) { + throw new TypeError(`expected fetch call ${callIndex} to receive a Request`); + } + return input; +} + +describe('unwrap / same-origin client', () => { + it('pins requests to same-origin /api/v1 and never api.html2rss.dev', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(metadataBody())); + + const envelope = await unwrap(getApiMetadata({ baseUrl: jsonBaseUrl() })); + const requested = httpRequestUrl(fetchRequest(fetchMock)); + + expect(requested.host).not.toBe('api.html2rss.dev'); + expect(requested.pathname).toMatch(/^\/api\/v1\/?$/); + expect(envelope).toMatchObject({ ok: true, status: 200, body: { success: true } }); + }); + + it('unwraps HTTP errors into ok/status/body without throwing', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ error: 'catalog_disabled' }, { status: 404 }) + ); + + const envelope = await unwrap(getApiMetadata({ baseUrl: jsonBaseUrl() })); + expect(envelope).toMatchObject({ + ok: false, + status: 404, + body: { error: 'catalog_disabled' }, + }); + }); +}); + +describe('requestApiMetadata', () => { + it('loads GET /api/v1 through the generated client on the same origin', async () => { + const body = metadataBody(); + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body)); + + const payload = await requestApiMetadata(); + const requested = httpRequestUrl(fetchRequest(fetchMock)); + + expect(requested.pathname).toMatch(/^\/api\/v1\/?$/); + expect(payload?.data.instance.feed_creation.enabled).toBe(true); + }); + + it('returns undefined when instance is missing', async () => { + const body = metadataBody(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ success: true, data: { api: body.data.api } }) + ); + + await expect(requestApiMetadata()).resolves.toBeUndefined(); + }); +}); + +describe('requestConfigCatalog', () => { + it('loads GET /configs through the generated client and ignores catalog.url', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(emptyCatalog)); + + const payload = await requestConfigCatalog(); + const requested = httpRequestUrl(fetchRequest(fetchMock)); + + expect(requested.host).not.toBe('api.html2rss.dev'); + expect(requested.pathname).toBe('/api/v1/configs'); + expect(payload).toMatchObject({ success: true, meta: { catalog_version: 2 } }); + }); + + it('returns undefined when the catalog is disabled', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ error: 'catalog_disabled' }, { status: 404 }) + ); + + await expect(requestConfigCatalog()).resolves.toBeUndefined(); + }); +}); + +describe('postCreateFeed', () => { + it('POSTs /feeds on the same origin and omits Authorization when the token is empty', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(Response.json({ success: false, error: { code: 'UNAUTHORIZED' } }, { status: 401 })); + + const envelope = await postCreateFeed('https://example.com', ' '); + const input = fetchRequest(fetchMock); + const requested = httpRequestUrl(input); + + expect(requested.pathname).toBe('/api/v1/feeds'); + expect(input.method).toBe('POST'); + expect(input.headers.get('Authorization')).toBeNull(); + expect(JSON.parse(await input.clone().text())).toEqual({ url: 'https://example.com' }); + expect(envelope).toMatchObject({ ok: false, status: 401 }); + }); + + it('sends Bearer auth and optional selectors through the generated client', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + Response.json( + createdFeedBody({ feed_token: 'tok', id: 'feed-1', name: 'Example', url: 'https://example.com' }), + { status: 201 } + ) + ); + const selectors = { + items: { selector: 'article' }, + title: { selector: 'h2', extractor: 'text' as const }, + }; + + const envelope = await postCreateFeed('https://example.com/articles', 'token-123', selectors); + const input = fetchRequest(fetchMock); + + expect(input.headers.get('Authorization')).toBe('Bearer token-123'); + expect(JSON.parse(await input.clone().text())).toEqual({ + url: 'https://example.com/articles', + selectors, + }); + expect(envelope).toMatchObject({ ok: true, status: 201 }); + }); +}); + +describe('studio JSON writes', () => { + it('POSTs validate, preview, and suggest through the generated client', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json({ success: true })); + const selectors = { items: { selector: 'article' } }; + + await postValidateFeed('token-123', 'https://example.com', selectors); + await postPreviewFeed('token-123', 'https://example.com', selectors); + await postSuggestSelectors('token-123', 'https://example.com'); + + const paths = [0, 1, 2].map((index) => httpRequestUrl(fetchRequest(fetchMock, index)).pathname); + + expect(paths).toEqual([ + '/api/v1/feeds/validate', + '/api/v1/feeds/preview', + '/api/v1/feeds/suggest_selectors', + ]); + }); +}); diff --git a/frontend/src/__tests__/appRoute.test.ts b/frontend/src/__tests__/appRoute.test.ts index 0f7c3b5f3..4ccc3fb39 100644 --- a/frontend/src/__tests__/appRoute.test.ts +++ b/frontend/src/__tests__/appRoute.test.ts @@ -19,6 +19,18 @@ describe('appRoute', () => { ); }); + it('reads and builds the unresolved workspace hash without a feed token', () => { + expect( + readAppRoute({ + pathname: '/', + search: '', + hash: '#/result', + }) + ).toEqual({ kind: 'result' }); + + expect(buildAppRouteHref({ kind: 'result' }, 'http://localhost/')).toBe('http://localhost/#/result'); + }); + it('keeps create and token prefill on their own variants only', () => { expect( readAppRoute({ @@ -51,12 +63,16 @@ describe('appRoute', () => { ).toEqual({ kind: 'create', prefillUrl: 'https://example.com/articles' }); }); - it('canonicalizes hashbang create hashes to #/create and bumps createEntryKey', () => { - history.replaceState({}, '', 'http://localhost:3000/#!/create'); + it('canonicalizes hashbang create hashes, preserves prefill, and bumps createEntryKey', () => { + history.replaceState({}, '', 'http://localhost:3000/#!/create?url=https%3A%2F%2Fexample.com%2Farticles'); const { result, unmount } = renderHook(() => useAppRoute()); - expect(location.hash).toBe('#/create'); + expect(location.hash).toBe('#/create?url=https%3A%2F%2Fexample.com%2Farticles'); + expect(result.current.route).toEqual({ + kind: 'create', + prefillUrl: 'https://example.com/articles', + }); expect(result.current.createEntryKey).toBeGreaterThan(0); unmount(); }); @@ -76,16 +92,21 @@ describe('appRoute', () => { unmount(); }); - it('canonicalizes hashbang create hashes with a url query', () => { - history.replaceState({}, '', 'http://localhost:3000/#!/create?url=https%3A%2F%2Fexample.com%2Farticles'); + it('treats an unknown hash as create and canonicalizes onto #/create', () => { + expect( + readAppRoute({ + pathname: '/', + search: '', + hash: '#/unknown?url=https%3A%2F%2Fexample.com', + }) + ).toEqual({ kind: 'create', prefillUrl: 'https://example.com' }); + + history.replaceState({}, '', 'http://localhost:3000/#/unknown'); const { result, unmount } = renderHook(() => useAppRoute()); - expect(location.hash).toBe('#/create?url=https%3A%2F%2Fexample.com%2Farticles'); - expect(result.current.route).toEqual({ - kind: 'create', - prefillUrl: 'https://example.com/articles', - }); + expect(location.hash).toBe('#/create'); + expect(result.current.route).toEqual({ kind: 'create' }); unmount(); }); }); diff --git a/frontend/src/__tests__/bookmarkletHref.test.ts b/frontend/src/__tests__/bookmarkletHref.test.ts new file mode 100644 index 000000000..293692dcc --- /dev/null +++ b/frontend/src/__tests__/bookmarkletHref.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { buildBookmarkletHref } from '../routes/bookmarkletHref'; + +describe('buildBookmarkletHref', () => { + it('builds a javascript assign to the create route with encoded current href', () => { + const href = buildBookmarkletHref('http://localhost:3000/'); + expect(href).toBe( + 'javascript:window.location.assign("http://localhost:3000/#/create?url="+encodeURIComponent(window.location.href));' + ); + expect(href).not.toContain('%27+encodeURIComponent'); + }); + + it('returns a hash placeholder when window is unavailable and no base is given', () => { + expect(buildBookmarkletHref()).toBe('#'); + }); +}); diff --git a/frontend/src/__tests__/decideJourney.test.ts b/frontend/src/__tests__/decideJourney.test.ts index 9e0ef9af1..f2d65dca4 100644 --- a/frontend/src/__tests__/decideJourney.test.ts +++ b/frontend/src/__tests__/decideJourney.test.ts @@ -3,6 +3,17 @@ import { decideJourney } from '../feed/decideJourney'; const emptyErrors = { url: '', form: '' }; +const readyFeed = { + id: 'feed-1', + name: 'Example', + url: 'https://example.com', + feed_token: 'token', + public_url: '/feed', + json_public_url: '/feed.json', + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-01-01T00:00:00Z', +}; + describe('decideJourney', () => { it('keeps create idle when there is no creation failure', () => { expect( @@ -15,23 +26,14 @@ describe('decideJourney', () => { ).toEqual({ kind: 'create' }); }); - it('nests preview under the result variant when tokens match', () => { + it('projects a ready result when tokens match', () => { const viewModel = decideJourney({ feedFieldErrors: emptyErrors, isCreating: false, route: { kind: 'result', feedToken: 'token' }, tokenError: '', result: { - feed: { - id: 'feed-1', - name: 'Example', - url: 'https://example.com', - feed_token: 'token', - public_url: '/feed', - json_public_url: '/feed.json', - created_at: '2024-01-01T00:00:00Z', - updated_at: '2024-01-01T00:00:00Z', - }, + feed: readyFeed, preview: { status: 'preview_ready', items: [], isLoading: false }, warnings: [], }, @@ -39,11 +41,12 @@ describe('decideJourney', () => { expect(viewModel).toMatchObject({ kind: 'result', + phase: 'ready', preview: { status: 'preview_ready' }, }); }); - it('does not treat a mismatched or missing result as the result kind', () => { + it('keeps the ready panel when the in-memory token leads the hash', () => { expect( decideJourney({ feedFieldErrors: emptyErrors, @@ -51,21 +54,12 @@ describe('decideJourney', () => { route: { kind: 'result', feedToken: 'route-token' }, tokenError: '', result: { - feed: { - id: 'feed-1', - name: 'Example', - url: 'https://example.com', - feed_token: 'other-token', - public_url: '/feed', - json_public_url: '/feed.json', - created_at: '2024-01-01T00:00:00Z', - updated_at: '2024-01-01T00:00:00Z', - }, + feed: { ...readyFeed, feed_token: 'other-token' }, preview: { status: 'preview_ready', items: [], isLoading: false }, warnings: [], }, }) - ).toEqual({ kind: 'create' }); + ).toMatchObject({ kind: 'result', phase: 'ready', feed: { feed_token: 'other-token' } }); expect( decideJourney({ @@ -77,6 +71,64 @@ describe('decideJourney', () => { ).toEqual({ kind: 'create' }); }); + it('projects unresolved workspace on #/result without a feed token', () => { + expect( + decideJourney({ + feedFieldErrors: emptyErrors, + isCreating: false, + route: { kind: 'result' }, + tokenError: '', + unresolved: { + url: 'https://example.com/articles', + notice: 'We could not extract feed items from this page yet.', + }, + }) + ).toEqual({ + kind: 'result', + phase: 'unresolved', + url: 'https://example.com/articles', + notice: 'We could not extract feed items from this page yet.', + }); + }); + + it('does not recover empty extraction onto result without unresolved memory', () => { + expect( + decideJourney({ + feedFieldErrors: emptyErrors, + isCreating: false, + route: { kind: 'result' }, + tokenError: '', + creationError: { + kind: 'input', + code: 'EXTRACTION_EMPTY', + retryable: false, + nextAction: 'correct_input', + retryAction: 'none', + message: 'Empty.', + }, + }) + ).toEqual({ kind: 'create' }); + }); + + it('keeps EXTRACTION_EMPTY as a create error when unresolved is absent', () => { + expect( + decideJourney({ + feedFieldErrors: emptyErrors, + isCreating: false, + route: { kind: 'create' }, + tokenError: '', + creationError: { + kind: 'input', + code: 'EXTRACTION_EMPTY', + retryable: false, + nextAction: 'correct_input', + retryAction: 'none', + message: 'Empty.', + }, + }) + ).toMatchObject({ kind: 'error', message: 'Empty.', errorKind: 'input' }); + }); + it('maps submitting and corrective input failures', () => { expect( decideJourney({ diff --git a/frontend/src/__tests__/directoryHandoff.test.ts b/frontend/src/__tests__/directoryHandoff.test.ts new file mode 100644 index 000000000..a5c803930 --- /dev/null +++ b/frontend/src/__tests__/directoryHandoff.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { buildDirectoryIssueBody, buildDirectoryIssueHref } from '../studio/directoryHandoff'; + +describe('directoryHandoff', () => { + it('builds a GitHub issue URL with channel, selectors, and blank directory fields', () => { + const channelUrl = 'https://example.com/articles'; + const selectorsYaml = `channel:\n url: ${channelUrl}\nselectors:\n items:\n selector: article\n`; + const href = buildDirectoryIssueHref(channelUrl, selectorsYaml); + const url = new URL(href); + + expect(url.origin + url.pathname).toBe('https://github.com/html2rss/html2rss-configs/issues/new'); + expect(url.searchParams.get('title')).toBe(`Add feed: ${channelUrl}`); + + const body = url.searchParams.get('body') ?? ''; + expect(body).toContain('channel:'); + expect(body).toContain(`url: ${channelUrl}`); + expect(body).toContain('selectors:'); + expect(body).toContain('selector: article'); + expect(body).toContain('directory:'); + expect(body).toContain('title:'); + expect(body).toContain('topics:'); + expect(body).not.toMatch(/access.?token/i); + expect(body).not.toMatch(/feed.?token/i); + expect(body).not.toContain('Authorization'); + }); + + it('falls back to channel.url when studio yaml is empty', () => { + const body = buildDirectoryIssueBody('https://example.com/list', ''); + expect(body).toBe( + 'channel:\n url: https://example.com/list\nselectors: {}\n\ndirectory:\n title:\n topics:\n' + ); + }); +}); diff --git a/frontend/src/__tests__/feedWorkflowStorage.test.ts b/frontend/src/__tests__/feedWorkflowStorage.test.ts index c4ca6af91..e310273fe 100644 --- a/frontend/src/__tests__/feedWorkflowStorage.test.ts +++ b/frontend/src/__tests__/feedWorkflowStorage.test.ts @@ -1,10 +1,10 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { clearFeedDraftState, loadFeedDraftState, saveFeedDraftState } from '../utils/feedWorkflowStorage'; +import { getPersistentStorage } from '../utils/persistentStorage'; describe('feedWorkflowStorage', () => { beforeEach(() => { - localStorage.clear(); - sessionStorage.clear(); + getPersistentStorage().clear(); }); it('persists and hydrates the create draft state from the url only', () => { @@ -13,16 +13,13 @@ describe('feedWorkflowStorage', () => { expect(loadFeedDraftState()).toEqual({ url: 'https://example.com/articles', }); - expect(localStorage.getItem('html2rss_feed_draft_state')).toBe( - JSON.stringify({ url: 'https://example.com/articles' }) - ); clearFeedDraftState(); expect(loadFeedDraftState()).toBeUndefined(); }); it('ignores extra draft properties beyond the canonical shape', () => { - localStorage.setItem( + getPersistentStorage().setItem( 'html2rss_feed_draft_state', JSON.stringify({ url: 'https://example.com/articles', diff --git a/frontend/src/__tests__/mocks/apiFixtures.ts b/frontend/src/__tests__/mocks/apiFixtures.ts new file mode 100644 index 000000000..7ecf8e7d3 --- /dev/null +++ b/frontend/src/__tests__/mocks/apiFixtures.ts @@ -0,0 +1,259 @@ +/** Pure API wire fixtures shared by MSW and Playwright. */ + +export type MetadataOptions = { + isTokenRequired?: boolean; + isCatalogEnabled?: boolean; + isStudioEnabled?: boolean; + openapiUrl?: string; +}; + +export function metadataBody({ + isTokenRequired = true, + isCatalogEnabled = true, + isStudioEnabled = true, + openapiUrl = 'https://example.test/openapi.yaml', +}: MetadataOptions = {}) { + return { + success: true, + data: { + api: { + name: 'html2rss-web API', + description: 'RESTful API for converting websites to RSS feeds', + openapi_url: openapiUrl, + }, + instance: { + feed_creation: { enabled: true, access_token_required: isTokenRequired }, + studio: { enabled: isStudioEnabled }, + catalog: { + enabled: isCatalogEnabled, + url: 'https://api.html2rss.dev/api/v1/configs', + }, + }, + }, + }; +} + +export const emptyCatalog = { + success: true, + data: { configs: [] }, + meta: { total: 0, catalog_version: 2, starters: [] }, +}; + +export type CatalogWireEntry = { + id: string; + path?: string; + channelUrl: string; + title: string; + summary?: string; + parameterDefaults?: Record; + lastResultState?: string; +}; + +export function catalogWireEntry({ + id, + path, + channelUrl, + title, + summary, + parameterDefaults = {}, + lastResultState = 'unknown', +}: CatalogWireEntry) { + return { + id, + path: path ?? `/${id}.rss`, + channel: { url: channelUrl }, + directory: { title, summary: summary ?? `${title} description.` }, + parameters: { schema: {}, defaults: parameterDefaults }, + last_result: { state: lastResultState }, + }; +} + +export function catalogBody(entries: ReturnType[], starters: string[] = []) { + return { + success: true, + data: { configs: entries }, + meta: { total: entries.length, catalog_version: 2, starters }, + }; +} + +export type CreatedFeedOverrides = { + id?: string; + name?: string; + url?: string; + feed_token?: string; + public_url?: string; + json_public_url?: string; + created_at?: string; + updated_at?: string; +}; + +export function createdFeedBody(overrides: CreatedFeedOverrides = {}) { + const timestamp = overrides.created_at ?? '2026-04-05T08:59:00.000Z'; + const token = overrides.feed_token ?? 'generated-token'; + return { + success: true, + data: { + feed: { + id: overrides.id ?? 'feed-123', + name: overrides.name ?? 'Example Feed', + url: overrides.url ?? 'https://example.com/articles', + feed_token: token, + public_url: overrides.public_url ?? `/api/v1/feeds/${token}`, + json_public_url: overrides.json_public_url ?? `/api/v1/feeds/${token}.json`, + created_at: timestamp, + updated_at: overrides.updated_at ?? '2026-04-05T09:00:00.000Z', + }, + }, + }; +} + +export function buildFeedResponse(overrides: CreatedFeedOverrides = {}) { + return { ...createdFeedBody(overrides), meta: { created: true } }; +} + +export type StructuredErrorOverrides = { + code?: string; + message?: string; + kind?: 'auth' | 'input' | 'network' | 'server'; + retryable?: boolean; + next_action?: 'enter_token' | 'correct_input' | 'retry' | 'wait' | 'none'; + retry_action?: 'alternate' | 'primary' | 'none'; +}; + +export function buildStructuredErrorResponse(overrides: StructuredErrorOverrides = {}) { + return { + success: false, + error: { + code: overrides.code ?? 'INTERNAL_SERVER_ERROR', + message: overrides.message ?? 'Internal Server Error', + kind: overrides.kind ?? 'server', + retryable: overrides.retryable ?? false, + next_action: overrides.next_action ?? 'none', + retry_action: overrides.retry_action ?? 'none', + }, + }; +} + +export function pageUrlFrom(body: unknown): string | undefined { + if (typeof body === 'object' && body && 'url' in body && typeof body.url === 'string' && body.url.trim()) { + return body.url; + } + return undefined; +} + +export function selectorsFromValidateRequest(body: unknown): unknown { + if (typeof body === 'object' && body && 'selectors' in body && body.selectors !== undefined) { + return body.selectors; + } + return { items: { selector: 'a[href]', enhance: false } }; +} + +export function validateYamlForSelectors(selectors: unknown, url?: string): string { + const channel = url ? `channel:\n url: ${url}\n` : ''; + if ( + typeof selectors === 'object' && + selectors && + 'items' in selectors && + typeof selectors.items === 'object' && + selectors.items && + 'selector' in selectors.items && + typeof selectors.items.selector === 'string' + ) { + const enhance = + 'enhance' in selectors.items && selectors.items.enhance === true ? '\n enhance: true' : ''; + return `${channel}selectors:\n items:\n selector: ${selectors.items.selector}${enhance}\n`; + } + return `${channel}selectors: {}\n`; +} + +export function validateSuccessBody(selectors?: unknown, yaml?: string) { + const resolved = selectors ?? { items: { selector: 'article' } }; + return { + success: true, + data: { + report: { success: true, issues: [] }, + selectors: resolved, + yaml: yaml ?? validateYamlForSelectors(resolved), + }, + }; +} + +export type SuggestCandidateFixture = { + selector: string; + enhance?: boolean; + sample?: string; +}; + +export type SuggestBuckets = { + items?: SuggestCandidateFixture[]; + title?: SuggestCandidateFixture[]; + link?: SuggestCandidateFixture[]; + published?: SuggestCandidateFixture[]; +}; + +function mapSuggestField(entries: SuggestCandidateFixture[] = []) { + return entries.map((entry) => ({ + selector: entry.selector, + sample: entry.sample ?? entry.selector, + })); +} + +export function suggestSuccessBody(buckets: SuggestBuckets = {}, extras: Record = {}) { + return { + success: true, + data: { + candidates: { + items: (buckets.items ?? []).map((entry) => ({ + selector: entry.selector, + enhance: entry.enhance === true, + sample: entry.sample ?? entry.selector, + })), + title: mapSuggestField(buckets.title), + link: mapSuggestField(buckets.link), + published: mapSuggestField(buckets.published), + }, + segment_strategy: 'list', + admission_drops: { chrome: 0 }, + ...extras, + }, + }; +} + +export const DEFAULT_SUGGEST_BUCKETS: SuggestBuckets = { + items: [{ selector: 'article', enhance: true, sample: 'Sample headline' }], +}; + +export function studioPreviewSuccessBody( + items: Array> = [ + { + title: 'Studio meadow item', + url: 'https://example.com/studio-item', + published_at: '2024-01-02', + }, + ] +) { + return { + success: true, + data: { + item_count: items.length, + sample_items: items, + quality_report: { warnings: [], metrics: { short_title_count: 0 } }, + // eslint-disable-next-line unicorn/no-null -- wire absence is null + failure_kind: null, + validation_issues: [], + }, + }; +} + +export const jsonPreviewItems = { + items: [ + { + title: 'Sample preview item', + content_text: 'Current preview fetch includes rendered content.', + date_published: '2026-04-05T09:00:00.000Z', + url: 'https://example.com/articles/sample-preview-item', + }, + ], +}; + +export const JSON_FEED_PREVIEW_PATH = /\/api\/v1\/feeds\/[^/?#]+\.json$/; diff --git a/frontend/src/__tests__/mocks/server.ts b/frontend/src/__tests__/mocks/server.ts index 87a103379..e2c9ed3af 100644 --- a/frontend/src/__tests__/mocks/server.ts +++ b/frontend/src/__tests__/mocks/server.ts @@ -1,88 +1,50 @@ import { setupServer } from 'msw/node'; import { http, HttpResponse } from 'msw'; +import { + emptyCatalog, + JSON_FEED_PREVIEW_PATH, + jsonPreviewItems, + metadataBody, + pageUrlFrom, + selectorsFromValidateRequest, + DEFAULT_SUGGEST_BUCKETS, + studioPreviewSuccessBody, + suggestSuccessBody, + validateSuccessBody, + validateYamlForSelectors, + type CreatedFeedOverrides, +} from './apiFixtures'; export const server = setupServer( - http.get('/api/v1', () => { - return HttpResponse.json({ - success: true, - data: { - api: { - name: 'html2rss-web API', - description: 'RESTful API for converting websites to RSS feeds', - openapi_url: 'https://example.test/openapi.yaml', - }, - instance: { - feed_creation: { - enabled: true, - access_token_required: true, - }, - catalog: { - enabled: true, - url: '/api/v1/configs', - }, - }, - }, - }); + http.get(/\/api\/v1\/?$/, () => { + return HttpResponse.json(metadataBody({ openapiUrl: 'https://example.test/openapi.yaml' })); }), http.get('/api/v1/configs', () => { - return HttpResponse.json({ - success: true, - data: { configs: [] }, - meta: { total: 0, catalog_version: 2, starters: [] }, - }); + return HttpResponse.json(emptyCatalog); + }), + http.get(JSON_FEED_PREVIEW_PATH, () => + HttpResponse.json(jsonPreviewItems, { + headers: { 'content-type': 'application/feed+json' }, + }) + ), + http.post('/api/v1/feeds/validate', async ({ request }) => { + const body = await request.json(); + const selectors = selectorsFromValidateRequest(body); + const yaml = validateYamlForSelectors(selectors, pageUrlFrom(body)); + return HttpResponse.json(validateSuccessBody(selectors, yaml)); + }), + http.post('/api/v1/feeds/preview', () => { + return HttpResponse.json(studioPreviewSuccessBody([])); + }), + http.post('/api/v1/feeds/suggest_selectors', () => { + return HttpResponse.json(suggestSuccessBody(DEFAULT_SUGGEST_BUCKETS)); }) ); -export interface FeedResponseOverrides { - id?: string; - name?: string; - url?: string; - feed_token?: string; - public_url?: string; - json_public_url?: string; - created_at?: string; - updated_at?: string; -} - -export interface StructuredErrorOverrides { - code?: string; - message?: string; - kind?: 'auth' | 'input' | 'network' | 'server'; - retryable?: boolean; - next_action?: 'enter_token' | 'correct_input' | 'retry' | 'wait' | 'none'; - retry_action?: 'alternate' | 'primary' | 'none'; -} - -export function buildFeedResponse(overrides: FeedResponseOverrides = {}) { - const timestamp = overrides.created_at ?? new Date('2024-01-01T00:00:00Z').toISOString(); - return { - success: true, - data: { - feed: { - id: overrides.id ?? 'feed-123', - name: overrides.name ?? 'Example Feed', - url: overrides.url ?? 'https://example.com/articles', - feed_token: overrides.feed_token ?? 'example-token', - public_url: overrides.public_url ?? '/api/v1/feeds/example-token', - json_public_url: overrides.json_public_url ?? '/api/v1/feeds/example-token.json', - created_at: timestamp, - updated_at: overrides.updated_at ?? timestamp, - }, - }, - meta: { created: true }, - }; -} +export type FeedResponseOverrides = CreatedFeedOverrides; -export function buildStructuredErrorResponse(overrides: StructuredErrorOverrides = {}) { - return { - success: false, - error: { - code: overrides.code ?? 'INTERNAL_SERVER_ERROR', - message: overrides.message ?? 'Internal Server Error', - kind: overrides.kind ?? 'server', - retryable: overrides.retryable ?? false, - next_action: overrides.next_action ?? 'none', - retry_action: overrides.retry_action ?? 'none', - }, - }; -} +export { + buildFeedResponse, + buildStructuredErrorResponse, + type StructuredErrorOverrides, +} from './apiFixtures'; diff --git a/frontend/src/__tests__/persistentStorage.test.ts b/frontend/src/__tests__/persistentStorage.test.ts new file mode 100644 index 000000000..18b971640 --- /dev/null +++ b/frontend/src/__tests__/persistentStorage.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { getPersistentStorage } from '../utils/persistentStorage'; + +describe('getPersistentStorage', () => { + afterEach(() => { + Reflect.deleteProperty(globalThis, 'window'); + Reflect.deleteProperty(globalThis, 'localStorage'); + Reflect.deleteProperty(globalThis, 'sessionStorage'); + }); + + it('uses the in-memory store when window is unavailable', () => { + const storage = getPersistentStorage(); + storage.clear(); + storage.setItem('k', 'v'); + expect(storage.getItem('k')).toBe('v'); + storage.removeItem('k'); + expect(storage.getItem('k')).toBeNull(); + }); + + it('falls back to memory when localStorage and sessionStorage throw on access', () => { + Object.defineProperties(globalThis, { + window: { value: globalThis, configurable: true }, + localStorage: { + configurable: true, + get() { + throw new Error('blocked'); + }, + }, + sessionStorage: { + configurable: true, + get() { + throw new Error('blocked'); + }, + }, + }); + + const storage = getPersistentStorage(); + storage.clear(); + storage.setItem('memory-key', 'memory-value'); + expect(storage.getItem('memory-key')).toBe('memory-value'); + }); +}); diff --git a/frontend/src/__tests__/previewHydration.test.ts b/frontend/src/__tests__/previewHydration.test.ts index 783086a59..f1f6a136d 100644 --- a/frontend/src/__tests__/previewHydration.test.ts +++ b/frontend/src/__tests__/previewHydration.test.ts @@ -43,7 +43,13 @@ describe('previewHydration', () => { it('normalizes json feed items and caps at five', () => { const items = normalizePreviewItems([ - { title: 'One', content_text: 'A', date_published: '2024-01-01', url: 'https://example.com/1' }, + { + title: 'One', + content_text: 'A', + date_published: '2024-01-01', + url: 'https://example.com/1', + image: 'https://cdn.example/a.jpg', + }, { title: 'Two', description: 'B', publishedLabel: 'Jan 2' }, { title: 'Three' }, { title: 'Four' }, @@ -60,6 +66,31 @@ describe('previewHydration', () => { publishedLabel: '2024-01-01', url: 'https://example.com/1', }); + expect(items[0]).not.toHaveProperty('imageUrl'); + }); + + it('keeps ten member items and only safe http(s) images', () => { + const items = normalizePreviewItems( + [ + { title: 'Safe', image: 'https://cdn.example/a.jpg' }, + // HTTP is an accepted image scheme; the assertion below must stay non-HTTPS. + // eslint-disable-next-line unicorn/prefer-https + { title: 'Plain http', image: 'http://cdn.example/b.jpg' }, + { title: 'Script', image: 'javascript:alert(1)' }, + { title: 'Data', image: 'data:image/png;base64,aaaa' }, + { title: 'Spaced', image: 'https://cdn.example/a b.jpg' }, + ...Array.from({ length: 7 }, (_, index) => ({ title: `Extra ${index}` })), + ], + 'member' + ); + + expect(items).toHaveLength(10); + expect(items[0]?.imageUrl).toBe('https://cdn.example/a.jpg'); + // eslint-disable-next-line unicorn/prefer-https -- same accepted HTTP image as the fixture + expect(items[1]?.imageUrl).toBe('http://cdn.example/b.jpg'); + expect(items[2]).not.toHaveProperty('imageUrl'); + expect(items[3]).not.toHaveProperty('imageUrl'); + expect(items[4]).not.toHaveProperty('imageUrl'); }); it('marks non-transient preview HTTP failures as non-retryable', async () => { diff --git a/frontend/src/__tests__/selectorDraft.test.ts b/frontend/src/__tests__/selectorDraft.test.ts new file mode 100644 index 000000000..f1063f8a5 --- /dev/null +++ b/frontend/src/__tests__/selectorDraft.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest'; +import { + draftFromSelectors, + emptyDraft, + hasEverySelectorPart, + selectorsFromDraft, + splitSelectorParts, + toggleSelectorParts, + withFieldSelector, +} from '../studio/selectorDraft'; + +describe('selectorDraft', () => { + it('starts the items selector at the gem default a[href]', () => { + expect(emptyDraft().itemsSelector).toBe('a[href]'); + }); + + it('splits on commas outside parens, brackets, and quotes, and drops empty parts', () => { + expect(splitSelectorParts('a[href], article, div')).toEqual(['a[href]', 'article', 'div']); + expect(splitSelectorParts('article,, div')).toEqual(['article', 'div']); + expect(splitSelectorParts(' article , div ')).toEqual(['article', 'div']); + expect(splitSelectorParts('div:not(.a, .b), article')).toEqual(['div:not(.a, .b)', 'article']); + expect(splitSelectorParts('div:has(a[href]), article')).toEqual(['div:has(a[href])', 'article']); + expect(splitSelectorParts('a[href="foo, bar"], main')).toEqual(['a[href="foo, bar"]', 'main']); + expect(splitSelectorParts("a[href='foo, bar'], main")).toEqual(["a[href='foo, bar']", 'main']); + expect(splitSelectorParts('')).toEqual([]); + }); + + it('treats a candidate as used when every part is present, not only on a whole-string match', () => { + expect(hasEverySelectorPart('a[href], article, div, main', 'article, div')).toBe(true); + expect(hasEverySelectorPart('article, div', 'article, div, main')).toBe(false); + expect(hasEverySelectorPart('article', 'article, div')).toBe(false); + }); + + it('adds candidate parts in order, dedupes exact parts, and removes one part without the others', () => { + const added = toggleSelectorParts(toggleSelectorParts('a[href]', 'article, div'), 'main'); + expect(added).toBe('a[href], article, div, main'); + expect(hasEverySelectorPart(added, 'article, div')).toBe(true); + expect(hasEverySelectorPart(added, 'main')).toBe(true); + + const removed = toggleSelectorParts(added, 'article'); + expect(removed).toBe('a[href], div, main'); + expect(removed.includes('article')).toBe(false); + + expect(toggleSelectorParts('article', 'article, div')).toBe('article, div'); + expect(toggleSelectorParts('a[href], article, div', 'article, div')).toBe('a[href]'); + }); + + it('refuses to hydrate documents outside StudioSelectors', () => { + expect( + draftFromSelectors({ + items: { selector: 'article', enhance: true, order: 'reverse' }, + guid: ['title'], + title: { selector: 'h2', extractor: 'text' }, + }) + ).toBeUndefined(); + expect(draftFromSelectors({ items: { selector: 'article', pagination: 2 } })).toBeUndefined(); + expect( + draftFromSelectors({ + items: { selector: 'article' }, + title: { + selector: 'h2', + extractor: 'text', + post_process: [{ name: 'gsub', pattern: 'x', replacement: '' }], + }, + }) + ).toBeUndefined(); + }); + + it('always emits enhance boolean so uncheck disables gem enrichment', () => { + const unchecked = emptyDraft(); + expect(unchecked.enhance).toBe(false); + expect(selectorsFromDraft(unchecked)).toEqual({ + items: { selector: 'a[href]', enhance: false }, + }); + + const checked = { ...unchecked, enhance: true }; + expect(selectorsFromDraft(checked)).toEqual({ + items: { selector: 'a[href]', enhance: true }, + }); + }); + + it('round-trips pure StudioSelectors without silent drop', () => { + const drafted = draftFromSelectors({ + items: { selector: 'article', enhance: true }, + title: { selector: 'h2', extractor: 'text' }, + }); + expect(drafted).toBeDefined(); + if (!drafted) return; + + expect(drafted.itemsSelector).toBe('article'); + expect(drafted.enhance).toBe(true); + expect(drafted.fields.title.selector).toBe('h2'); + + const withLink = withFieldSelector(drafted, 'link', 'a'); + const withDate = withFieldSelector(withLink, 'published', 'time'); + const wire = selectorsFromDraft(withDate); + expect(wire).toEqual({ + items: { selector: 'article', enhance: true }, + title: { selector: 'h2', extractor: 'text' }, + url: { selector: 'a', extractor: 'href' }, + published_at: { selector: 'time', extractor: 'text' }, + }); + + const again = draftFromSelectors(wire); + expect(again?.fields.title.selector).toBe('h2'); + expect(again?.fields.link.selector).toBe('a'); + expect(again?.fields.published.selector).toBe('time'); + expect(selectorsFromDraft(again ?? withDate)).toEqual(wire); + }); +}); diff --git a/frontend/src/__tests__/setup.ts b/frontend/src/__tests__/setup.ts index f7e7f3af8..50014eb11 100644 --- a/frontend/src/__tests__/setup.ts +++ b/frontend/src/__tests__/setup.ts @@ -1,100 +1,40 @@ -import '@testing-library/jest-dom'; -import { afterAll, afterEach, beforeAll, beforeEach, vi } from 'vitest'; +import '@testing-library/jest-dom/vitest'; +import { afterAll, afterEach, beforeAll, vi } from 'vitest'; import { cleanup } from '@testing-library/preact'; +import { server } from './mocks/server'; -let server: typeof import('./mocks/server').server; - -// Mock window and document for tests Object.defineProperty(globalThis, 'matchMedia', { writable: true, - value: vi.fn().mockImplementation((query) => ({ + value: (query: string) => ({ matches: false, media: query, onchange: undefined, - addListener: vi.fn(), // deprecated - removeListener: vi.fn(), // deprecated - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - dispatchEvent: vi.fn(), - })), -}); - -// Persistent storage stubs with in-memory backing store -const createStorageMock = () => { - const store = new Map(); - - return { - store, - api: { - get length() { - return store.size; - }, - // eslint-disable-next-line unicorn/no-null -- Web Storage returns null for missing keys. - getItem: vi.fn((key: string) => (store.has(key) ? store.get(key)! : null)), - setItem: vi.fn((key: string, value: string) => { - store.set(key, value); - }), - removeItem: vi.fn((key: string) => { - store.delete(key); - }), - clear: vi.fn(() => { - store.clear(); - }), - // eslint-disable-next-line unicorn/no-null, unicorn/prefer-iterator-to-array -- Web Storage key() returns null for out-of-range indexes. - key: vi.fn((index: number) => [...store.keys()][index] ?? null), - }, - }; -}; - -const local = createStorageMock(); -const session = createStorageMock(); - -Object.defineProperties(globalThis, { - localStorage: { - value: local.api, - configurable: true, - writable: true, - }, - sessionStorage: { - value: session.api, - configurable: true, - writable: true, - }, + addListener: () => {}, + removeListener: () => {}, + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => false, + }), }); -beforeEach(() => { - local.store.clear(); - session.store.clear(); - local.api.getItem.mockClear(); - local.api.setItem.mockClear(); - local.api.removeItem.mockClear(); - local.api.clear.mockClear(); - local.api.key.mockClear(); - session.api.getItem.mockClear(); - session.api.setItem.mockClear(); - session.api.removeItem.mockClear(); - session.api.clear.mockClear(); - session.api.key.mockClear(); +const clipboardWriteText = vi.fn(() => Promise.resolve()); +Object.defineProperty(navigator, 'clipboard', { + configurable: true, + value: { writeText: clipboardWriteText }, }); -// Mock clipboard API -Object.assign(navigator, { - clipboard: { - writeText: vi.fn(() => Promise.resolve()), - }, -}); - -// Ensure scrollIntoView exists for components relying on it Element.prototype.scrollIntoView = vi.fn(); -// Wire up MSW in node environment -beforeAll(async () => { - // eslint-disable-next-line unicorn/no-top-level-assignment-in-function - ({ server } = await import('./mocks/server')); +beforeAll(() => { server.listen({ onUnhandledRequest: 'error' }); }); + afterEach(() => { server.resetHandlers(); cleanup(); + clipboardWriteText.mockClear(); +}); + +afterAll(() => { + server.close(); }); -afterAll(() => server.close()); diff --git a/frontend/src/__tests__/setup.unit.ts b/frontend/src/__tests__/setup.unit.ts new file mode 100644 index 000000000..9c9e0a393 --- /dev/null +++ b/frontend/src/__tests__/setup.unit.ts @@ -0,0 +1,5 @@ +/** Minimal DOM location for Node unit tests that exercise same-origin HTTP adapters. */ +Object.defineProperty(globalThis, 'location', { + value: { origin: 'http://localhost:3000' }, + configurable: true, +}); diff --git a/frontend/src/__tests__/studioModel.test.ts b/frontend/src/__tests__/studioModel.test.ts new file mode 100644 index 000000000..a7613ba5f --- /dev/null +++ b/frontend/src/__tests__/studioModel.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest'; +import { emptyDraft, withFieldSelector } from '../studio/selectorDraft'; +import { + groupStudioIssues, + normalizePreviewFailure, + dormantStudioFields, + studioFieldSurfaces, + studioSavePresentation, + suggestionResult, + type StudioCandidates, + type StudioIssue, +} from '../studio/studioModel'; + +const emptyCandidates: StudioCandidates = { + items: [], + fields: { title: [], link: [], published: [] }, +}; + +describe('studioModel', () => { + it('separates a valid empty suggestion from ranked candidates', () => { + expect(suggestionResult(emptyCandidates)).toEqual({ status: 'empty' }); + + const candidates: StudioCandidates = { + ...emptyCandidates, + fields: { + ...emptyCandidates.fields, + title: [{ selector: 'h2', sample: 'Headline' }], + }, + }; + expect(suggestionResult(candidates)).toEqual({ status: 'ready', candidates }); + }); + + it('groups issues by the editor control that owns them', () => { + const issues: readonly StudioIssue[] = [ + { path: ['selectors', 'items', 'selector'], code: 'constraint', message: 'items' }, + { path: ['selectors', 'items', 'enhance'], code: 'constraint', message: 'enhance' }, + { path: ['selectors', 'url', 'selector'], code: 'constraint', message: 'link' }, + { path: ['selectors', 'pagination'], code: 'constraint', message: 'hidden' }, + ]; + + const grouped = groupStudioIssues(issues); + + expect(grouped.items.map(({ message }) => message)).toEqual(['items']); + expect(grouped.enhance.map(({ message }) => message)).toEqual(['enhance']); + expect(grouped.fields.link.map(({ message }) => message)).toEqual(['link']); + expect(grouped.hidden.map(({ message }) => message)).toEqual(['hidden']); + }); + + it('projects optional fields from selector, candidate, issue, and manual evidence', () => { + const draft = withFieldSelector(emptyDraft(), 'title', 'h2'); + const suggestion = suggestionResult({ + ...emptyCandidates, + fields: { + ...emptyCandidates.fields, + link: [{ selector: 'a', sample: 'Link' }], + }, + }); + const issues = groupStudioIssues([ + { path: ['selectors', 'published_at'], code: 'missing_key', message: 'published' }, + ]); + + expect(studioFieldSurfaces(draft, suggestion, issues, new Set(['link']))).toEqual({ + title: { status: 'visible', evidence: ['selector'] }, + link: { status: 'visible', evidence: ['candidate', 'manual'] }, + published: { status: 'visible', evidence: ['issue'] }, + }); + expect(studioFieldSurfaces(emptyDraft(), { status: 'empty' }, groupStudioIssues([]))).toEqual({ + title: { status: 'dormant' }, + link: { status: 'dormant' }, + published: { status: 'dormant' }, + }); + const emptySurfaces = studioFieldSurfaces(emptyDraft(), { status: 'empty' }, groupStudioIssues([])); + expect(dormantStudioFields(emptySurfaces)).toEqual(['title', 'link', 'published']); + }); + + it('owns ready and unresolved Save presentation', () => { + expect(studioSavePresentation('unresolved', true, false)).toEqual({ + enabled: true, + className: 'btn btn--primary', + }); + expect(studioSavePresentation('ready', true, false)).toEqual({ + enabled: false, + className: 'btn btn--ghost', + }); + expect(studioSavePresentation('ready', true, true)).toEqual({ + enabled: true, + className: 'btn btn--primary', + }); + }); + + it('normalizes known and future preview failure codes without casting', () => { + expect(normalizePreviewFailure(undefined)).toBeUndefined(); + expect(normalizePreviewFailure('min_items')).toEqual({ kind: 'known', code: 'min_items' }); + expect(normalizePreviewFailure('future_kind')).toEqual({ kind: 'unknown', code: 'future_kind' }); + }); +}); diff --git a/frontend/src/__tests__/studioService.test.ts b/frontend/src/__tests__/studioService.test.ts new file mode 100644 index 000000000..c931ff4d7 --- /dev/null +++ b/frontend/src/__tests__/studioService.test.ts @@ -0,0 +1,132 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { postPreviewFeed, postSuggestSelectors } from '../api/http/feeds'; +import { COPY } from '../journey/copy'; +import { emptyDraft } from '../studio/selectorDraft'; +import { previewStudioFeed, StudioRequestError, suggestStudioSelectors } from '../studio/studioService'; + +vi.mock('../api/http/feeds', () => ({ + postPreviewFeed: vi.fn(), + postSuggestSelectors: vi.fn(), + postValidateFeed: vi.fn(), +})); + +const pageUrl = 'https://example.com/articles'; +const token = 'studio-token'; + +function okEnvelope(data: unknown) { + return { ok: true, status: 200, body: { success: true, data } }; +} + +async function fieldOnFailure(work: Promise): Promise { + try { + await work; + } catch (error) { + expect(error).toBeInstanceOf(StudioRequestError); + if (!(error instanceof StudioRequestError)) return; + expect(error.message).toBe(COPY.unableToCompleteCreation); + return error.field; + } + expect.fail('expected a studio parse error'); +} + +describe('studio payload parsing', () => { + beforeEach(() => { + vi.mocked(postPreviewFeed).mockReset(); + vi.mocked(postSuggestSelectors).mockReset(); + }); + + it('names sample_items when the preview sample list is not an array', async () => { + vi.mocked(postPreviewFeed).mockResolvedValue(okEnvelope({ item_count: 0, sample_items: 'nope' })); + + const field = await fieldOnFailure(previewStudioFeed(token, pageUrl, emptyDraft())); + expect(field).toBe('sample_items'); + }); + + it('names sample_items when one sample is missing a title', async () => { + vi.mocked(postPreviewFeed).mockResolvedValue( + okEnvelope({ item_count: 1, sample_items: [{ url: 'https://example.com/a' }] }) + ); + + const field = await fieldOnFailure(previewStudioFeed(token, pageUrl, emptyDraft())); + expect(field).toBe('sample_items'); + }); + + it('names candidates when the suggestion buckets are missing', async () => { + vi.mocked(postSuggestSelectors).mockResolvedValue(okEnvelope({})); + + const field = await fieldOnFailure(suggestStudioSelectors(token, pageUrl)); + expect(field).toBe('candidates'); + }); + + it('returns a ready candidate result without leaking wire enhance', async () => { + vi.mocked(postSuggestSelectors).mockResolvedValue( + okEnvelope({ + candidates: { + items: [{ selector: 'article', enhance: true, sample: 'Story' }], + title: [{ selector: 'h2', sample: 'Headline' }], + link: [], + published: [], + }, + }) + ); + + await expect(suggestStudioSelectors(token, pageUrl)).resolves.toEqual({ + status: 'ready', + candidates: { + items: [{ selector: 'article', sample: 'Story' }], + fields: { + title: [{ selector: 'h2', sample: 'Headline' }], + link: [], + published: [], + }, + }, + }); + }); + + it('returns an explicit empty result for valid empty buckets', async () => { + vi.mocked(postSuggestSelectors).mockResolvedValue( + okEnvelope({ candidates: { items: [], title: [], link: [], published: [] } }) + ); + + await expect(suggestStudioSelectors(token, pageUrl)).resolves.toEqual({ status: 'empty' }); + }); + + it('rejects malformed item enhance instead of leaking or applying it', async () => { + vi.mocked(postSuggestSelectors).mockResolvedValue( + okEnvelope({ + candidates: { + items: [{ selector: 'article', enhance: 'yes', sample: 'Story' }], + title: [], + link: [], + published: [], + }, + }) + ); + + expect(await fieldOnFailure(suggestStudioSelectors(token, pageUrl))).toBe('candidates'); + }); + + it.each([ + [401, 'Authentication required'], + [403, 'Studio is disabled'], + [429, 'Too many requests'], + ])('preserves the server decision for a %i suggestion response', async (status, message) => { + vi.mocked(postSuggestSelectors).mockResolvedValue({ + ok: false, + status, + body: { success: false, error: { message } }, + }); + + await expect(suggestStudioSelectors(token, pageUrl)).rejects.toThrow(message); + }); + + it('uses owned fallback copy when a failed suggestion has no message', async () => { + vi.mocked(postSuggestSelectors).mockResolvedValue({ + ok: false, + status: 500, + body: { success: false, error: {} }, + }); + + await expect(suggestStudioSelectors(token, pageUrl)).rejects.toThrow(COPY.unableToCompleteCreation); + }); +}); diff --git a/frontend/src/__tests__/useFeedCreation.contract.test.ts b/frontend/src/__tests__/useFeedCreation.contract.test.ts deleted file mode 100644 index 69f4ca4e2..000000000 --- a/frontend/src/__tests__/useFeedCreation.contract.test.ts +++ /dev/null @@ -1,214 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { renderHook, act, waitFor } from '@testing-library/preact'; -import { http, HttpResponse } from 'msw'; -import { server, buildFeedResponse, buildStructuredErrorResponse } from './mocks/server'; -import { useFeedCreation } from '../feed'; - -describe('useFeedCreation contract', () => { - it('sends feed creation requests with bearer auth and hydrates preview from json_public_url', async () => { - let receivedAuthorization: string | undefined; - const nativeFetch = fetch; - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation((input, init) => { - if (String(input).endsWith('/api/v1/feeds/generated-token.json')) { - return Promise.resolve( - Response.json( - { items: [{ title: 'Preview', content_text: 'Text' }] }, - { - status: 200, - headers: { 'Content-Type': 'application/feed+json' }, - } - ) - ); - } - - return nativeFetch(input, init); - }); - - server.use( - http.post('/api/v1/feeds', async ({ request }) => { - const body = (await request.json()) as { url: string }; - receivedAuthorization = request.headers.get('authorization'); - - expect(body).toEqual({ url: 'https://example.com/articles' }); - - return HttpResponse.json( - buildFeedResponse({ - url: body.url, - feed_token: 'generated-token', - public_url: '/api/v1/feeds/generated-token', - json_public_url: '/api/v1/feeds/generated-token.json', - }), - { status: 201 } - ); - }), - http.get('http://localhost:3000/api/v1/feeds/generated-token.json', () => - HttpResponse.json({ items: [{ title: 'Preview', content_text: 'Text' }] }) - ), - http.get('http://localhost/api/v1/feeds/generated-token.json', () => - HttpResponse.json({ items: [{ title: 'Preview', content_text: 'Text' }] }) - ), - http.get('/api/v1/feeds/generated-token.json', () => - HttpResponse.json({ items: [{ title: 'Preview', content_text: 'Text' }] }) - ) - ); - - const { result } = renderHook(() => useFeedCreation()); - - await act(async () => { - await result.current.createFeed('https://example.com/articles', 'test-token-123'); - }); - - expect(receivedAuthorization).toBe('Bearer test-token-123'); - expect(result.current.error).toBeUndefined(); - expect(result.current.result?.feed.feed_token).toBe('generated-token'); - await waitFor(() => expect(result.current.result?.preview.status).toBe('preview_ready')); - fetchSpy.mockRestore(); - }); - - it('propagates structured auth failures without parsing the message text', async () => { - server.use( - http.post('/api/v1/feeds', async () => - HttpResponse.json( - buildStructuredErrorResponse({ - code: 'UNAUTHORIZED', - message: 'Authentication required', - kind: 'auth', - retryable: false, - next_action: 'enter_token', - retry_action: 'none', - }), - { status: 401 } - ) - ) - ); - - const { result } = renderHook(() => useFeedCreation()); - - await act(async () => { - await expect(result.current.createFeed('https://example.com/articles', 'token')).rejects.toMatchObject({ - message: 'Authentication required', - }); - }); - - expect(result.current.result).toBeUndefined(); - expect(result.current.error).toMatchObject({ - kind: 'auth', - code: 'UNAUTHORIZED', - nextAction: 'enter_token', - retryAction: 'none', - retryable: false, - message: 'Authentication required', - }); - }); - - it('treats extraction-empty failures as corrective input errors without strategy metadata', async () => { - server.use( - http.post('/api/v1/feeds', async () => - HttpResponse.json( - buildStructuredErrorResponse({ - code: 'NO_FEED_ITEMS_EXTRACTED', - message: 'Could not extract feed items. Try a more specific listing URL or explicit selectors.', - kind: 'input', - retryable: false, - next_action: 'correct_input', - retry_action: 'none', - }), - { status: 422 } - ) - ) - ); - - const { result } = renderHook(() => useFeedCreation()); - - await act(async () => { - await expect(result.current.createFeed('https://example.com/articles', 'token')).rejects.toMatchObject({ - kind: 'input', - code: 'NO_FEED_ITEMS_EXTRACTED', - nextAction: 'correct_input', - retryAction: 'none', - retryable: false, - message: 'Could not extract feed items. Try a more specific listing URL or explicit selectors.', - }); - }); - }); - - it('marks preview failure from the feed json response without status polling', async () => { - const nativeFetch = fetch; - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation((input, init) => { - if (String(input).endsWith('/api/v1/feeds/generated-token.json')) { - return Promise.resolve(new Response('No feed items', { status: 422 })); - } - - return nativeFetch(input, init); - }); - - server.use( - http.post('/api/v1/feeds', async ({ request }) => { - const body = (await request.json()) as { url: string }; - - return HttpResponse.json( - buildFeedResponse({ - url: body.url, - feed_token: 'generated-token', - public_url: '/api/v1/feeds/generated-token', - json_public_url: '/api/v1/feeds/generated-token.json', - }), - { status: 201 } - ); - }), - http.get('http://localhost:3000/api/v1/feeds/generated-token.json', () => - HttpResponse.text('No feed items', { status: 422 }) - ), - http.get('http://localhost/api/v1/feeds/generated-token.json', () => - HttpResponse.text('No feed items', { status: 422 }) - ), - http.get('/api/v1/feeds/generated-token.json', () => - HttpResponse.text('No feed items', { status: 422 }) - ) - ); - - const { result } = renderHook(() => useFeedCreation()); - - await act(async () => { - await result.current.createFeed('https://example.com/articles', 'token'); - }); - - await waitFor(() => { - expect(result.current.result?.preview.status).toBe('preview_failed'); - expect(result.current.result?.warnings[0]?.code).toBe('PREVIEW_HTTP_422'); - }); - fetchSpy.mockRestore(); - }); - - it('rejects camelCase-only create payloads to enforce canonical snake_case contract', async () => { - server.use( - http.post('/api/v1/feeds', async () => - HttpResponse.json( - { - success: true, - data: { - feed: { - id: 'feed-1', - name: 'Example Feed', - url: 'https://example.com/articles', - feedToken: 'generated-token', - publicUrl: '/api/v1/feeds/generated-token', - jsonPublicUrl: '/api/v1/feeds/generated-token.json', - }, - }, - }, - { status: 201 } - ) - ) - ); - - const { result } = renderHook(() => useFeedCreation()); - - await act(async () => { - await expect(result.current.createFeed('https://example.com/articles', 'token')).rejects.toMatchObject({ - kind: 'server', - code: 'INVALID_RESPONSE', - }); - }); - }); -}); diff --git a/frontend/src/__tests__/useFeedCreation.test.ts b/frontend/src/__tests__/useFeedCreation.test.ts index 6a9974080..01d905ab0 100644 --- a/frontend/src/__tests__/useFeedCreation.test.ts +++ b/frontend/src/__tests__/useFeedCreation.test.ts @@ -1,5 +1,8 @@ -import { describe, it, expect, beforeEach, afterEach, vi, type SpyInstance } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { renderHook, act, waitFor } from '@testing-library/preact'; +import { http, HttpResponse } from 'msw'; +import { server, buildFeedResponse, buildStructuredErrorResponse } from './mocks/server'; +import { JSON_FEED_PREVIEW_PATH, jsonPreviewItems } from './mocks/apiFixtures'; import { useFeedCreation } from '../feed'; const mockFeed = { @@ -13,63 +16,56 @@ const mockFeed = { updated_at: '2024-01-01T00:00:00Z', }; -function createResponse(status = 201) { - return Response.json( - { success: true, data: { feed: mockFeed } }, - { - status, - headers: { 'Content-Type': 'application/json' }, - } - ); -} - -function previewResponse(status = 200) { - return Response.json( - { - items: [{ title: 'Preview item', content_text: 'Preview excerpt', date_published: '2024-01-02' }], - }, - { status, headers: { 'Content-Type': 'application/feed+json' } } +function useCreateFeedHandler() { + let createCount = 0; + server.use( + http.post('/api/v1/feeds', () => { + createCount += 1; + return HttpResponse.json(buildFeedResponse(mockFeed), { status: 201 }); + }) ); + return () => createCount; } describe('useFeedCreation', () => { - let fetchMock: SpyInstance; - beforeEach(() => { - vi.clearAllMocks(); localStorage.clear(); sessionStorage.clear(); - fetchMock = vi.spyOn(globalThis, 'fetch'); }); afterEach(() => { - fetchMock.mockRestore(); + vi.useRealTimers(); }); it('creates a feed from metadata and hydrates preview from json_public_url', async () => { - fetchMock.mockResolvedValueOnce(createResponse()).mockResolvedValueOnce(previewResponse()); + useCreateFeedHandler(); const { result } = renderHook(() => useFeedCreation()); await act(async () => { await result.current.createFeed('https://example.com/articles', 'token-123'); }); - expect(fetchMock.mock.calls[0]?.[0]).toBe('/api/v1/feeds'); - expect(String(fetchMock.mock.calls[1]?.[0])).toMatch(/\/api\/v1\/feeds\/feed-token-1\.json$/); - expect(fetchMock.mock.calls).toHaveLength(2); - await waitFor(() => { expect(result.current.result?.preview.status).toBe('preview_ready'); - expect(result.current.result?.preview.items[0]?.title).toBe('Preview item'); + expect(result.current.result?.preview.items[0]?.title).toBe(jsonPreviewItems.items[0]?.title); }); }); it('retries only preview fetches on transient preview failure', async () => { vi.useFakeTimers(); - fetchMock - .mockResolvedValueOnce(createResponse()) - .mockResolvedValueOnce(new Response('', { status: 503 })) - .mockResolvedValueOnce(previewResponse()); + const createCount = useCreateFeedHandler(); + let previewAttempts = 0; + server.use( + http.get(JSON_FEED_PREVIEW_PATH, () => { + previewAttempts += 1; + if (previewAttempts === 1) { + return new HttpResponse('', { status: 503 }); + } + return HttpResponse.json(jsonPreviewItems, { + headers: { 'content-type': 'application/feed+json' }, + }); + }) + ); const { result } = renderHook(() => useFeedCreation()); await act(async () => { @@ -83,50 +79,132 @@ describe('useFeedCreation', () => { await waitFor(() => { expect(result.current.result?.preview.status).toBe('preview_ready'); }); - expect(fetchMock.mock.calls.filter((call) => String(call[0]) === '/api/v1/feeds').length).toBe(1); - expect( - fetchMock.mock.calls.filter((call) => String(call[0]).endsWith('/api/v1/feeds/feed-token-1.json')) - .length - ).toBe(2); - - vi.useRealTimers(); + expect(createCount()).toBe(1); + expect(previewAttempts).toBe(2); }); - it('marks preview failed for non-transient preview responses', async () => { - fetchMock - .mockResolvedValueOnce(createResponse()) - .mockResolvedValueOnce(new Response('', { status: 422 })); + it('retries preview without recreating the feed', async () => { + const createCount = useCreateFeedHandler(); + let previewAttempts = 0; + server.use( + http.get(JSON_FEED_PREVIEW_PATH, () => { + previewAttempts += 1; + if (previewAttempts === 1) { + return new HttpResponse('', { status: 422 }); + } + return HttpResponse.json(jsonPreviewItems, { + headers: { 'content-type': 'application/feed+json' }, + }); + }) + ); const { result } = renderHook(() => useFeedCreation()); await act(async () => { await result.current.createFeed('https://example.com/articles', 'token-123'); }); + await waitFor(() => expect(result.current.result?.preview.status).toBe('preview_failed')); - await waitFor(() => { - expect(result.current.result?.preview.status).toBe('preview_failed'); - expect(result.current.result?.warnings[0]).toMatchObject({ - code: 'PREVIEW_HTTP_422', - retryable: false, - nextAction: 'wait', - }); - }); + act(() => result.current.retryPreviewFetch()); + + await waitFor(() => expect(result.current.result?.preview.status).toBe('preview_ready')); + expect(createCount()).toBe(1); }); - it('retries preview without recreating the feed', async () => { - fetchMock - .mockResolvedValueOnce(createResponse()) - .mockResolvedValueOnce(new Response('', { status: 422 })); + it('keeps the creation audience when preview is retried', async () => { + useCreateFeedHandler(); + let previewAttempts = 0; + const richPreview = { + items: [ + { title: 'Shown', image: 'https://cdn.example/a.jpg' }, + ...Array.from({ length: 9 }, (_, index) => ({ title: `Extra ${index}` })), + ], + }; + server.use( + http.get(JSON_FEED_PREVIEW_PATH, () => { + previewAttempts += 1; + if (previewAttempts === 1) { + return new HttpResponse('', { status: 422 }); + } + return HttpResponse.json(richPreview, { + headers: { 'content-type': 'application/feed+json' }, + }); + }) + ); const { result } = renderHook(() => useFeedCreation()); await act(async () => { - await result.current.createFeed('https://example.com/articles', 'token-123'); + await result.current.createFeed('https://example.com/articles', ' '.repeat(3)); }); await waitFor(() => expect(result.current.result?.preview.status).toBe('preview_failed')); - fetchMock.mockResolvedValueOnce(previewResponse()); act(() => result.current.retryPreviewFetch()); await waitFor(() => expect(result.current.result?.preview.status).toBe('preview_ready')); - expect(fetchMock.mock.calls.filter((call) => String(call[0]) === '/api/v1/feeds').length).toBe(1); + expect(result.current.result?.preview.items).toHaveLength(5); + expect(result.current.result?.preview.items[0]).not.toHaveProperty('imageUrl'); + }); + + it('rejects camelCase-only create payloads to enforce canonical snake_case', async () => { + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + { + success: true, + data: { + feed: { + id: 'feed-1', + name: 'Example Feed', + url: 'https://example.com/articles', + feedToken: 'generated-token', + publicUrl: '/api/v1/feeds/generated-token', + jsonPublicUrl: '/api/v1/feeds/generated-token.json', + }, + }, + }, + { status: 201 } + ) + ) + ); + + const { result } = renderHook(() => useFeedCreation()); + + await act(async () => { + await expect(result.current.createFeed('https://example.com/articles', 'token')).rejects.toMatchObject({ + kind: 'server', + code: 'INVALID_RESPONSE', + }); + }); + }); + + it('propagates structured auth failures without parsing the message text', async () => { + server.use( + http.post('/api/v1/feeds', () => + HttpResponse.json( + buildStructuredErrorResponse({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none', + }), + { status: 401 } + ) + ) + ); + + const { result } = renderHook(() => useFeedCreation()); + + await act(async () => { + await expect(result.current.createFeed('https://example.com/articles', 'token')).rejects.toMatchObject({ + message: 'Authentication required', + }); + }); + + expect(result.current.error).toMatchObject({ + kind: 'auth', + code: 'UNAUTHORIZED', + nextAction: 'enter_token', + }); }); }); diff --git a/frontend/src/__tests__/useFeedFlow.test.ts b/frontend/src/__tests__/useFeedFlow.test.ts deleted file mode 100644 index b9e35a787..000000000 --- a/frontend/src/__tests__/useFeedFlow.test.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi, type SpyInstance } from 'vitest'; -import { renderHook, act, waitFor } from '@testing-library/preact'; -import { useFeedFlow, type FeedFlowDependencies } from '../feed'; - -const mockFeed = { - id: 'feed-1', - name: 'Example Feed', - url: 'https://example.com/articles', - feed_token: 'feed-token-1', - public_url: '/api/v1/feeds/feed-token-1', - json_public_url: '/api/v1/feeds/feed-token-1.json', - created_at: '2024-01-01T00:00:00Z', - updated_at: '2024-01-01T00:00:00Z', -}; - -async function stubFeedCreationFailure(message = 'Upstream failed') { - const { server, buildStructuredErrorResponse } = await import('./mocks/server'); - const { http, HttpResponse } = await import('msw'); - server.use( - http.post('/api/v1/feeds', () => - HttpResponse.json( - buildStructuredErrorResponse({ - message, - kind: 'server', - retryable: true, - next_action: 'retry', - retry_action: 'primary', - }), - { status: 500 } - ) - ) - ); -} - -describe('useFeedFlow', () => { - let fetchMock: SpyInstance; - const mockNavigate = vi.fn(); - const mockSaveToken = vi.fn(); - const mockClearToken = vi.fn(); - - const feedFlowProperties = (overrides: Partial = {}): FeedFlowDependencies => ({ - token: undefined, - isLoading: false, - feedCreationEnabled: true, - mayCreate: () => 'proceed', - saveToken: mockSaveToken, - clearToken: mockClearToken, - route: { kind: 'create' }, - navigate: mockNavigate, - createEntryKey: 0, - ...overrides, - }); - - beforeEach(() => { - vi.clearAllMocks(); - localStorage.clear(); - sessionStorage.clear(); - fetchMock = vi.spyOn(globalThis, 'fetch'); - }); - - afterEach(() => { - fetchMock.mockRestore(); - }); - - it('prefills the url field from route prefill even when a draft url exists', async () => { - localStorage.setItem( - 'html2rss_feed_draft_state', - JSON.stringify({ url: 'https://old.example.com/page' }) - ); - - const { result } = renderHook(() => - useFeedFlow( - feedFlowProperties({ - route: { kind: 'create', prefillUrl: 'https://example.com/articles' }, - }) - ) - ); - - await waitFor(() => { - expect(result.current.feedFormData.url).toBe('https://example.com/articles'); - }); - }); - - it('manages form input field changes and draft state', async () => { - const { result } = renderHook(() => useFeedFlow(feedFlowProperties())); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/new-articles'); - }); - - expect(result.current.feedFormData.url).toBe('https://example.com/new-articles'); - expect(JSON.parse(localStorage.getItem('html2rss_feed_draft_state') || '{}').url).toBe( - 'https://example.com/new-articles' - ); - }); - - it('performs validation and creates feed on submit when no token is required', async () => { - fetchMock - .mockResolvedValueOnce(Response.json({ success: true, data: { feed: mockFeed } })) - .mockResolvedValueOnce(Response.json({ items: [] })); - - const { result } = renderHook(() => useFeedFlow(feedFlowProperties())); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/new-articles'); - }); - - const event = { preventDefault: vi.fn() } as any; - await act(async () => { - await result.current.onFeedSubmit(event); - }); - - expect(fetchMock).toHaveBeenCalled(); - expect(mockNavigate).toHaveBeenCalledWith({ kind: 'result', feedToken: 'feed-token-1' }); - }); - - it('redirects to token route when token is required but missing', async () => { - const { result } = renderHook(() => - useFeedFlow( - feedFlowProperties({ - mayCreate: () => 'needToken', - }) - ) - ); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/private-articles'); - }); - - const event = { preventDefault: vi.fn() } as any; - await act(async () => { - await result.current.onFeedSubmit(event); - }); - - expect(fetchMock).not.toHaveBeenCalled(); - expect(mockNavigate).toHaveBeenCalledWith({ - kind: 'token', - prefillUrl: 'https://example.com/private-articles', - }); - }); - - it('navigates to token and projects token_prompt on auth creation failure', async () => { - const { server, buildStructuredErrorResponse } = await import('./mocks/server'); - const { http, HttpResponse } = await import('msw'); - server.use( - http.post('/api/v1/feeds', () => - HttpResponse.json( - buildStructuredErrorResponse({ - message: 'Access denied', - kind: 'auth', - code: 'UNAUTHORIZED', - retryable: false, - next_action: 'enter_token', - retry_action: 'none', - }), - { status: 401 } - ) - ) - ); - - const { result, rerender } = renderHook( - ({ route }) => - useFeedFlow(feedFlowProperties({ route, mayCreate: () => 'proceed', token: 'bad-token' })), - { initialProps: { route: { kind: 'create' as const } } } - ); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/private-articles'); - }); - - await act(async () => { - await result.current.onFeedSubmit({ preventDefault: vi.fn() } as any); - }); - - expect(mockNavigate).toHaveBeenCalledWith({ - kind: 'token', - prefillUrl: 'https://example.com/private-articles', - }); - expect(mockClearToken).toHaveBeenCalled(); - - rerender({ route: { kind: 'token', prefillUrl: 'https://example.com/private-articles' } }); - - expect(result.current.viewModel.kind).toBe('token_prompt'); - expect(result.current.tokenError).toBeTruthy(); - }); - - it('returns non-auth creation failures from token onto create', async () => { - await stubFeedCreationFailure(); - - const { result, rerender } = renderHook( - ({ route }) => - useFeedFlow( - feedFlowProperties({ - mayCreate: () => 'proceed', - route, - }) - ), - { - initialProps: { - route: { kind: 'token' as const, prefillUrl: 'https://example.com/private-articles' }, - }, - } - ); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/private-articles'); - result.current.setTokenDraft('token-123'); - }); - - mockSaveToken.mockResolvedValueOnce(undefined); - - await act(async () => { - await result.current.onSaveToken(); - }); - - expect(mockNavigate).toHaveBeenCalledWith({ - kind: 'create', - prefillUrl: 'https://example.com/private-articles', - }); - expect(result.current.tokenError).toBe(''); - expect(result.current.feedFieldErrors.form).toBe('Upstream failed'); - expect(result.current.creationError).toMatchObject({ - message: 'Upstream failed', - nextAction: 'retry', - }); - - // Simulate router applying token→create; remount must keep Decision error for retry chrome. - rerender({ - route: { kind: 'create', prefillUrl: 'https://example.com/private-articles' }, - }); - - expect(result.current.feedFieldErrors.form).toBe('Upstream failed'); - expect(result.current.creationError).toMatchObject({ - message: 'Upstream failed', - nextAction: 'retry', - }); - }); - - it('recovers unmatched result routes onto remounted create without prefillUrl', async () => { - const { result, rerender } = renderHook( - ({ route, createEntryKey }) => useFeedFlow(feedFlowProperties({ route, createEntryKey })), - { initialProps: { route: { kind: 'result' as const, feedToken: 'missing-token' }, createEntryKey: 0 } } - ); - - await waitFor(() => { - expect(mockNavigate).toHaveBeenCalledWith({ kind: 'create' }, { replace: true }); - }); - - for (const [route] of mockNavigate.mock.calls) { - if (route.kind !== 'create') continue; - expect(route).toEqual({ kind: 'create' }); - expect(route).not.toHaveProperty('prefillUrl'); - } - - rerender({ route: { kind: 'create' }, createEntryKey: 0 }); - - expect(result.current.focusCreateComposerKey).toBeGreaterThan(0); - expect(result.current.creationError).toBeUndefined(); - }); - - it('remounts create on a later create-entry visit and clears creation chrome', async () => { - await stubFeedCreationFailure(); - - const { result, rerender } = renderHook( - ({ createEntryKey }) => useFeedFlow(feedFlowProperties({ createEntryKey })), - { initialProps: { createEntryKey: 0 } } - ); - - act(() => { - result.current.onFeedFieldChange('url', 'https://example.com/articles'); - }); - - await act(async () => { - await result.current.onFeedSubmit({ preventDefault: vi.fn() } as any); - }); - - expect(result.current.creationError).toMatchObject({ message: 'Upstream failed' }); - expect(result.current.feedFieldErrors.form).toBe('Upstream failed'); - const previousFocusKey = result.current.focusCreateComposerKey; - const fetchCallsAfterSubmit = fetchMock.mock.calls.length; - - rerender({ createEntryKey: 1 }); - - expect(result.current.creationError).toBeUndefined(); - expect(result.current.feedFieldErrors.form).toBe(''); - expect(result.current.tokenError).toBe(''); - expect(result.current.tokenDraft).toBe(''); - expect(result.current.focusCreateComposerKey).toBeGreaterThan(previousFocusKey); - expect(fetchMock).toHaveBeenCalledTimes(fetchCallsAfterSubmit); - }); - - it('clears token draft on cancel and create remount', async () => { - const { result, rerender } = renderHook( - ({ createEntryKey, route }) => useFeedFlow(feedFlowProperties({ createEntryKey, route })), - { - initialProps: { - createEntryKey: 0, - route: { kind: 'token' as const, prefillUrl: 'https://example.com' }, - }, - } - ); - - act(() => { - result.current.setTokenDraft('secret-draft'); - }); - expect(result.current.tokenDraft).toBe('secret-draft'); - - act(() => { - result.current.onCancelTokenPrompt(); - }); - expect(result.current.tokenDraft).toBe(''); - expect(mockNavigate).toHaveBeenCalledWith({ kind: 'create', prefillUrl: expect.any(String) }); - - act(() => { - result.current.setTokenDraft('another-secret'); - }); - rerender({ createEntryKey: 1, route: { kind: 'create' } }); - expect(result.current.tokenDraft).toBe(''); - }); -}); diff --git a/frontend/src/__tests__/useSession.test.ts b/frontend/src/__tests__/useSession.test.ts deleted file mode 100644 index 74ff83de4..000000000 --- a/frontend/src/__tests__/useSession.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi, type SpyInstance } from 'vitest'; -import { renderHook, act, waitFor } from '@testing-library/preact'; -import { resetAccessTokenMemory, useSession } from '../session'; -import { getPersistentStorage } from '../utils/persistentStorage'; - -const ACCESS_TOKEN_KEY = 'html2rss_access_token'; - -const mockMetadata = { - instance: { - feed_creation: { enabled: true, access_token_required: true }, - catalog: { enabled: true, url: '/api/v1/configs' }, - }, - api: { openapi_url: '/openapi.yaml' }, -}; - -describe('useSession', () => { - let fetchMock: SpyInstance; - - const emptyCatalogResponse = Response.json({ - success: true, - data: { configs: [] }, - meta: { total: 0, catalog_version: 2, starters: [] }, - }); - - const mockFetchFor = (metadata: unknown, catalogResponse: Response = emptyCatalogResponse) => { - fetchMock.mockImplementation(async (input: RequestInfo | URL) => { - const url = String(input); - if (url.includes('/configs')) return catalogResponse.clone(); - return Response.json({ success: true, data: metadata }); - }); - }; - - beforeEach(() => { - vi.clearAllMocks(); - getPersistentStorage().clear(); - localStorage.clear(); - sessionStorage.clear(); - resetAccessTokenMemory(); - fetchMock = vi.spyOn(globalThis, 'fetch'); - }); - - afterEach(() => { - resetAccessTokenMemory(); - getPersistentStorage().clear(); - fetchMock.mockRestore(); - }); - - it('coordinates api metadata load and token loading', async () => { - localStorage.setItem(ACCESS_TOKEN_KEY, 'session-token'); - mockFetchFor(mockMetadata); - - const { result } = renderHook(() => useSession()); - - expect(result.current.isLoading).toBe(true); - - await waitFor(() => { - expect(result.current.isLoading).toBe(false); - }); - - expect(result.current.token).toBe('session-token'); - expect(result.current.hasToken).toBe(true); - expect(result.current.featuredFeeds).toEqual([]); - expect(result.current.catalogEntries).toEqual([]); - expect(result.current.metadataError).toBeUndefined(); - expect(result.current.feedCreationEnabled).toBe(true); - }); - - it('selects starter feeds when feed creation is enabled', async () => { - const fao = { - id: 'fao.org/newsroom', - path: '/fao.org/newsroom.rss', - channel: { url: 'https://www.fao.org/newsroom' }, - directory: { title: 'FAO Newsroom', summary: 'News' }, - parameters: { defaults: {} }, - last_result: { state: 'unknown' }, - }; - mockFetchFor( - mockMetadata, - Response.json({ - success: true, - data: { configs: [fao] }, - meta: { total: 1, catalog_version: 2, starters: ['fao.org/newsroom'] }, - }) - ); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.feedCreationEnabled).toBe(true); - expect(result.current.featuredFeeds.map((entry) => entry.id)).toEqual(['fao.org/newsroom']); - }); - - it('saves new tokens to persistent storage and does not write sessionStorage', async () => { - mockFetchFor(mockMetadata); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - await act(async () => { - await result.current.saveToken('brand-new-token'); - }); - - expect(result.current.token).toBe('brand-new-token'); - expect(result.current.hasToken).toBe(true); - expect(getPersistentStorage().getItem(ACCESS_TOKEN_KEY)).toBe('brand-new-token'); - expect(localStorage.getItem(ACCESS_TOKEN_KEY)).toBe('brand-new-token'); - expect(sessionStorage.getItem(ACCESS_TOKEN_KEY)).toBeNull(); - }); - - it('clears the canonical persistent token copy', async () => { - localStorage.setItem(ACCESS_TOKEN_KEY, 'old-token'); - mockFetchFor(mockMetadata); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - act(() => { - result.current.clearToken(); - }); - - expect(result.current.token).toBeUndefined(); - expect(result.current.hasToken).toBe(false); - expect(getPersistentStorage().getItem(ACCESS_TOKEN_KEY)).toBeNull(); - expect(sessionStorage.getItem(ACCESS_TOKEN_KEY)).toBeNull(); - }); - - it('falls back to in-memory token when persistent storage write is unavailable', async () => { - mockFetchFor(mockMetadata); - localStorage.setItem.mockImplementationOnce(() => { - throw new Error('blocked'); - }); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - await act(async () => { - await result.current.saveToken('memory-token'); - }); - - expect(result.current.token).toBe('memory-token'); - expect(result.current.hasToken).toBe(true); - expect(sessionStorage.getItem(ACCESS_TOKEN_KEY)).toBeNull(); - }); - - it('loads from in-memory fallback when persistent storage read is unavailable', async () => { - mockFetchFor(mockMetadata); - localStorage.setItem.mockImplementationOnce(() => { - throw new Error('blocked'); - }); - - const seeded = renderHook(() => useSession()); - await waitFor(() => expect(seeded.result.current.isLoading).toBe(false)); - await act(async () => { - await seeded.result.current.saveToken('memory-only'); - }); - seeded.unmount(); - - localStorage.getItem.mockImplementationOnce(() => { - throw new Error('blocked'); - }); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.token).toBe('memory-only'); - expect(result.current.hasToken).toBe(true); - expect(sessionStorage.getItem(ACCESS_TOKEN_KEY)).toBeNull(); - }); - - it('mayCreate returns needToken when access token is required and missing', async () => { - mockFetchFor(mockMetadata); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.mayCreate()).toBe('needToken'); - expect(result.current.mayCreate('')).toBe('needToken'); - expect(result.current.mayCreate('provided-token')).toBe('proceed'); - }); - - it('mayCreate returns disabled when feed creation is disabled', async () => { - mockFetchFor({ - ...mockMetadata, - instance: { - ...mockMetadata.instance, - feed_creation: { enabled: false, access_token_required: true }, - }, - }); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.feedCreationEnabled).toBe(false); - expect(result.current.mayCreate('any-token')).toBe('disabled'); - }); - - it('mayCreate returns proceed when token is not required', async () => { - mockFetchFor({ - ...mockMetadata, - instance: { - ...mockMetadata.instance, - feed_creation: { enabled: true, access_token_required: false }, - }, - }); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.mayCreate()).toBe('proceed'); - }); - - it('defaults gate to token-required when metadata feed_creation is absent', async () => { - mockFetchFor({ - api: mockMetadata.api, - instance: {}, - }); - - const { result } = renderHook(() => useSession()); - await waitFor(() => expect(result.current.isLoading).toBe(false)); - - expect(result.current.feedCreationEnabled).toBe(true); - expect(result.current.mayCreate()).toBe('needToken'); - }); -}); diff --git a/frontend/src/__tests__/useStudio.test.ts b/frontend/src/__tests__/useStudio.test.ts new file mode 100644 index 000000000..a2c90fdfd --- /dev/null +++ b/frontend/src/__tests__/useStudio.test.ts @@ -0,0 +1,256 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { renderHook, act, waitFor } from '@testing-library/preact'; +import { http, HttpResponse } from 'msw'; +import { server } from './mocks/server'; +import { + buildStructuredErrorResponse, + studioPreviewSuccessBody, + suggestSuccessBody, + validateSuccessBody, + validateYamlForSelectors, +} from './mocks/apiFixtures'; +import { + STUDIO_PREVIEW_SETTLE_MS, + STUDIO_VALIDATE_DELAY_MS, + canSaveStudio, + useStudio, +} from '../studio/useStudio'; + +const pageUrl = 'https://example.com/articles'; +const token = 'studio-token'; + +function suggestItems(selector: string, sample = 'Sample headline') { + return suggestSuccessBody({ items: [{ selector, sample }] }); +} + +async function advance(ms: number) { + await act(async () => { + await vi.advanceTimersByTimeAsync(ms); + }); +} + +function useStudioHandlers() { + let validateCount = 0; + let previewCount = 0; + let lastValidateAuth: string | undefined; + let lastValidateSelectors: unknown; + + server.use( + http.post('/api/v1/feeds/suggest_selectors', () => HttpResponse.json(suggestItems('article'))), + http.post('/api/v1/feeds/validate', async ({ request }) => { + validateCount += 1; + lastValidateAuth = request.headers.get('authorization') ?? undefined; + const body = (await request.json()) as { selectors?: unknown; url?: string }; + const selectors = body.selectors ?? { items: { selector: 'a[href]' } }; + lastValidateSelectors = selectors; + return HttpResponse.json(validateSuccessBody(selectors, validateYamlForSelectors(selectors, body.url))); + }), + http.post('/api/v1/feeds/preview', () => { + previewCount += 1; + return HttpResponse.json( + studioPreviewSuccessBody([ + { title: 'Sample', url: 'https://example.com/sample', published_at: '2024-01-02' }, + ]) + ); + }) + ); + + return { + validateCount: () => validateCount, + previewCount: () => previewCount, + lastValidateAuth: () => lastValidateAuth, + lastValidateSelectors: () => lastValidateSelectors, + }; +} + +describe('useStudio', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('waits for validate delay, sends bearer token, and unwraps a valid report', async () => { + vi.useFakeTimers(); + const handlers = useStudioHandlers(); + const { result } = renderHook(() => useStudio({ url: pageUrl, token })); + + expect(handlers.validateCount()).toBe(0); + await act(async () => { + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + }); + expect(handlers.validateCount()).toBe(0); + + await advance(STUDIO_VALIDATE_DELAY_MS - 1); + expect(handlers.validateCount()).toBe(0); + await advance(1); + expect(handlers.validateCount()).toBe(1); + expect(handlers.lastValidateAuth()).toBe(`Bearer ${token}`); + expect(result.current.state).toMatchObject({ phase: 'editing', validated: true }); + }); + + it('previews after settle, sends enhance boolean, and clears hold on enhance flip', async () => { + vi.useFakeTimers(); + const handlers = useStudioHandlers(); + const { result } = renderHook(() => + useStudio({ url: pageUrl, token, validateDelayMs: STUDIO_VALIDATE_DELAY_MS }) + ); + + await advance(STUDIO_VALIDATE_DELAY_MS); + expect(handlers.lastValidateSelectors()).toEqual({ + items: { selector: 'a[href]', enhance: false }, + }); + expect(handlers.previewCount()).toBe(0); + + act(() => { + result.current.setItemsSelector('article'); + }); + await advance(STUDIO_VALIDATE_DELAY_MS); + await advance(STUDIO_PREVIEW_SETTLE_MS - 1); + expect(handlers.previewCount()).toBe(0); + await advance(1); + expect(handlers.previewCount()).toBe(1); + expect(result.current.state.live).toMatchObject({ + status: 'ready', + sample: { items: [{ title: 'Sample' }] }, + }); + + act(() => { + result.current.setEnhance(true); + }); + expect(result.current.state.draft.enhance).toBe(true); + expect(result.current.state.live.status).toBe('idle'); + await advance(STUDIO_VALIDATE_DELAY_MS); + expect(handlers.lastValidateSelectors()).toEqual({ + items: { selector: 'article', enhance: true }, + }); + }); + + it('ignores a stale save after a newer save starts', async () => { + vi.useFakeTimers(); + useStudioHandlers(); + const { result } = renderHook(() => useStudio({ url: pageUrl, token, validateDelayMs: 0 })); + await advance(0); + + const stale = Promise.withResolvers(); + const current = Promise.withResolvers(); + act(() => { + void result.current.save(() => stale.promise); + }); + act(() => { + void result.current.save(() => current.promise); + }); + expect(result.current.state.phase).toBe('saving'); + + await act(async () => { + stale.resolve(); + await stale.promise; + }); + expect(result.current.state.phase).toBe('saving'); + + await act(async () => { + current.resolve(); + await current.promise; + }); + expect(result.current.state).toMatchObject({ phase: 'editing', validated: true }); + expect(canSaveStudio(result.current.state)).toBe(true); + }); + + it('fails closed when validate is unauthorized', async () => { + server.use( + http.post('/api/v1/feeds/suggest_selectors', () => HttpResponse.json(suggestItems('article'))), + http.post('/api/v1/feeds/validate', () => + HttpResponse.json( + buildStructuredErrorResponse({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none', + }), + { status: 401 } + ) + ) + ); + + const { result } = renderHook(() => useStudio({ url: pageUrl, token, validateDelayMs: 0 })); + await waitFor(() => + expect(result.current.state).toMatchObject({ + phase: 'failed', + message: 'Authentication required', + }) + ); + }); + + it('distinguishes loading, empty, and ready suggestion outcomes', async () => { + const pending = Promise.withResolvers(); + useStudioHandlers(); + server.use(http.post('/api/v1/feeds/suggest_selectors', () => pending.promise)); + + const { result } = renderHook(() => useStudio({ url: pageUrl, token })); + expect(result.current.suggestion).toEqual({ status: 'loading' }); + + pending.resolve(HttpResponse.json(suggestItems('article'))); + await waitFor(() => expect(result.current.suggestion.status).toBe('ready')); + + server.use(http.post('/api/v1/feeds/suggest_selectors', () => HttpResponse.json(suggestSuccessBody({})))); + act(() => result.current.retrySuggestion()); + await waitFor(() => expect(result.current.suggestion).toEqual({ status: 'empty' })); + }); + + it('retries suggestion decisions and aborts stale in-flight suggests', async () => { + useStudioHandlers(); + let attempts = 0; + let firstSignal: AbortSignal | undefined; + const stale = Promise.withResolvers(); + server.use( + http.post('/api/v1/feeds/suggest_selectors', ({ request }) => { + attempts += 1; + if (attempts === 1) { + return HttpResponse.json( + buildStructuredErrorResponse({ + code: 'RATE_LIMITED', + message: 'Wait before trying selector choices again', + kind: 'server', + retryable: true, + next_action: 'wait', + retry_action: 'primary', + }), + { status: 429 } + ); + } + if (attempts === 2) { + firstSignal = request.signal; + return stale.promise; + } + return HttpResponse.json(suggestItems('section.current')); + }) + ); + + const { result } = renderHook(() => useStudio({ url: pageUrl, token })); + await waitFor(() => + expect(result.current.suggestion).toEqual({ + status: 'failed', + message: 'Wait before trying selector choices again', + }) + ); + + act(() => result.current.retrySuggestion()); + expect(result.current.suggestion).toEqual({ status: 'loading' }); + await waitFor(() => expect(attempts).toBe(2)); + + act(() => result.current.retrySuggestion()); + await waitFor(() => expect(firstSignal?.aborted).toBe(true)); + await waitFor(() => expect(result.current.suggestion.status).toBe('ready')); + + stale.resolve(HttpResponse.json(suggestItems('article.stale'))); + await act(async () => { + await Promise.resolve(); + }); + expect(result.current.suggestion).toMatchObject({ + status: 'ready', + candidates: { items: [{ selector: 'section.current' }] }, + }); + expect(attempts).toBe(3); + }); +}); diff --git a/frontend/src/api/contracts.ts b/frontend/src/api/contracts.ts index 76fffed96..cfd71b68c 100644 --- a/frontend/src/api/contracts.ts +++ b/frontend/src/api/contracts.ts @@ -32,11 +32,15 @@ export type FeedCreationErrorCode = | 'UNKNOWN_ERROR' | (string & {}); +/** Guest stays compact. Member is the token used when the feed was created. */ +export type PreviewAudience = 'guest' | 'member'; + export interface FeedPreviewItem { title: string; excerpt: string; publishedLabel: string; url?: string; + imageUrl?: string; } export interface FeedPreviewWarning { @@ -75,6 +79,9 @@ export interface ApiMetadataRecord { enabled: boolean; access_token_required: boolean; }; + studio?: { + enabled: boolean; + }; catalog?: { enabled: boolean; url: string; diff --git a/frontend/src/api/generated/client.gen.ts b/frontend/src/api/generated/client.gen.ts index 97c964c59..5276b9393 100644 --- a/frontend/src/api/generated/client.gen.ts +++ b/frontend/src/api/generated/client.gen.ts @@ -13,4 +13,4 @@ import type { ClientOptions as ClientOptions2 } from './types.gen'; */ export type CreateClientConfig = (override?: Config) => Config & T>; -export const client: Client = createClient(createConfig({ baseUrl: 'https://api.html2rss.dev/api/v1' })); +export const client: Client = createClient(createConfig({ baseUrl: '/api/v1' })); diff --git a/frontend/src/api/generated/index.ts b/frontend/src/api/generated/index.ts index 56b4fe60d..e94cc1616 100644 --- a/frontend/src/api/generated/index.ts +++ b/frontend/src/api/generated/index.ts @@ -1,4 +1,4 @@ // This file is auto-generated by @hey-api/openapi-ts -export { createFeed, getApiMetadata, getConfigCatalog, getHealthStatus, getLivenessProbe, getReadinessProbe, listStrategies, type Options, optionsConfigCatalog, renderFeedByToken } from './sdk.gen'; -export type { ClientOptions, CreateFeedData, CreateFeedError, CreateFeedErrors, CreateFeedResponse, CreateFeedResponses, GetApiMetadataData, GetApiMetadataResponse, GetApiMetadataResponses, GetConfigCatalogData, GetConfigCatalogError, GetConfigCatalogErrors, GetConfigCatalogResponse, GetConfigCatalogResponses, GetHealthStatusData, GetHealthStatusError, GetHealthStatusErrors, GetHealthStatusResponse, GetHealthStatusResponses, GetLivenessProbeData, GetLivenessProbeResponse, GetLivenessProbeResponses, GetReadinessProbeData, GetReadinessProbeResponse, GetReadinessProbeResponses, ListStrategiesData, ListStrategiesResponse, ListStrategiesResponses, OptionsConfigCatalogData, OptionsConfigCatalogResponse, OptionsConfigCatalogResponses, RenderFeedByTokenData, RenderFeedByTokenError, RenderFeedByTokenErrors, RenderFeedByTokenResponse, RenderFeedByTokenResponses } from './types.gen'; +export { createFeed, getApiMetadata, getConfigCatalog, getHealthStatus, getLivenessProbe, getReadinessProbe, listStrategies, type Options, optionsConfigCatalog, previewFeed, renderFeedByToken, suggestSelectors, validateFeedConfig } from './sdk.gen'; +export type { ClientOptions, CreateFeedData, CreateFeedError, CreateFeedErrors, CreateFeedResponse, CreateFeedResponses, FieldSelectorCandidate, GetApiMetadataData, GetApiMetadataResponse, GetApiMetadataResponses, GetConfigCatalogData, GetConfigCatalogError, GetConfigCatalogErrors, GetConfigCatalogResponse, GetConfigCatalogResponses, GetHealthStatusData, GetHealthStatusError, GetHealthStatusErrors, GetHealthStatusResponse, GetHealthStatusResponses, GetLivenessProbeData, GetLivenessProbeResponse, GetLivenessProbeResponses, GetReadinessProbeData, GetReadinessProbeResponse, GetReadinessProbeResponses, Html2RssExtractorAttribute, Html2RssExtractorHref, Html2RssExtractorHtml, Html2RssExtractorStatic, Html2RssExtractorText, Html2RssPostProcessorGsub, Html2RssPostProcessorMarkdownToHtml, Html2RssPostProcessorParseTime, Html2RssPostProcessorParseUri, Html2RssPostProcessorSanitizeHtml, Html2RssPostProcessorSubstring, Html2RssPostProcessorTemplate, ItemsSelectorCandidate, ListStrategiesData, ListStrategiesResponse, ListStrategiesResponses, OptionsConfigCatalogData, OptionsConfigCatalogResponse, OptionsConfigCatalogResponses, PreviewFeedData, PreviewFeedResponse, PreviewFeedResponses, PreviewSampleItem, RenderFeedByTokenData, RenderFeedByTokenError, RenderFeedByTokenErrors, RenderFeedByTokenResponse, RenderFeedByTokenResponses, SelectorCandidates, SelectorsDocument, SuggestSelectorsData, SuggestSelectorsError, SuggestSelectorsErrors, SuggestSelectorsResponse, SuggestSelectorsResponses, ValidateFeedConfigData, ValidateFeedConfigError, ValidateFeedConfigErrors, ValidateFeedConfigResponse, ValidateFeedConfigResponses } from './types.gen'; diff --git a/frontend/src/api/generated/sdk.gen.ts b/frontend/src/api/generated/sdk.gen.ts index 858a3abcf..2df975af7 100644 --- a/frontend/src/api/generated/sdk.gen.ts +++ b/frontend/src/api/generated/sdk.gen.ts @@ -2,7 +2,7 @@ import { type Client, type ClientMeta, type Options as Options2, type RequestResult, type TDataShape, urlSearchParamsBodySerializer } from './client'; import { client } from './client.gen'; -import type { CreateFeedData, CreateFeedErrors, CreateFeedResponses, GetApiMetadataData, GetApiMetadataResponses, GetConfigCatalogData, GetConfigCatalogErrors, GetConfigCatalogResponses, GetHealthStatusData, GetHealthStatusErrors, GetHealthStatusResponses, GetLivenessProbeData, GetLivenessProbeResponses, GetReadinessProbeData, GetReadinessProbeResponses, ListStrategiesData, ListStrategiesResponses, OptionsConfigCatalogData, OptionsConfigCatalogResponses, RenderFeedByTokenData, RenderFeedByTokenErrors, RenderFeedByTokenResponses } from './types.gen'; +import type { CreateFeedData, CreateFeedErrors, CreateFeedResponses, GetApiMetadataData, GetApiMetadataResponses, GetConfigCatalogData, GetConfigCatalogErrors, GetConfigCatalogResponses, GetHealthStatusData, GetHealthStatusErrors, GetHealthStatusResponses, GetLivenessProbeData, GetLivenessProbeResponses, GetReadinessProbeData, GetReadinessProbeResponses, ListStrategiesData, ListStrategiesResponses, OptionsConfigCatalogData, OptionsConfigCatalogResponses, PreviewFeedData, PreviewFeedResponses, RenderFeedByTokenData, RenderFeedByTokenErrors, RenderFeedByTokenResponses, SuggestSelectorsData, SuggestSelectorsErrors, SuggestSelectorsResponses, ValidateFeedConfigData, ValidateFeedConfigErrors, ValidateFeedConfigResponses } from './types.gen'; export type Options = Options2 & { /** @@ -62,6 +62,51 @@ export const createFeed = (options: Option } }); +/** + * Preview a selectors document feed + * + * Preview a selectors document feed + */ +export const previewFeed = (options: Options): RequestResult => (options.client ?? client).post({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/feeds/preview', + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers + } +}); + +/** + * Suggest selectors for a page + * + * Suggest selectors for a page + */ +export const suggestSelectors = (options: Options): RequestResult => (options.client ?? client).post({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/feeds/suggest_selectors', + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers + } +}); + +/** + * Validate a selectors document + * + * Validate a selectors document + */ +export const validateFeedConfig = (options: Options): RequestResult => (options.client ?? client).post({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/feeds/validate', + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers + } +}); + /** * Render feed by token * diff --git a/frontend/src/api/generated/types.gen.ts b/frontend/src/api/generated/types.gen.ts index f24843d17..fbcf58360 100644 --- a/frontend/src/api/generated/types.gen.ts +++ b/frontend/src/api/generated/types.gen.ts @@ -4,6 +4,230 @@ export type ClientOptions = { baseUrl: 'https://api.html2rss.dev/api/v1' | 'http://127.0.0.1:4000/api/v1' | (string & {}); }; +/** + * One ranked field selector with a sample headline. Best match is first. + */ +export type FieldSelectorCandidate = { + /** + * Visible text of a matched node + */ + sample: string; + selector: string; +}; + +/** + * attribute + * + * Return the value of an HTML attribute on the selected element. Requires sibling selector option `attribute` (attribute name). + */ +export type Html2RssExtractorAttribute = 'attribute'; + +/** + * href + * + * Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the page base URL). + */ +export type Html2RssExtractorHref = 'href'; + +/** + * html + * + * Return the outer HTML of the selected element. Sanitize during post-processing (e.g. `sanitize_html`). + */ +export type Html2RssExtractorHtml = 'html'; + +/** + * static + * + * Return a fixed value from sibling selector option `static` (no DOM read). + */ +export type Html2RssExtractorStatic = 'static'; + +/** + * text + * + * Return collapsed visible text of the selected element (default extractor). + */ +export type Html2RssExtractorText = 'text'; + +/** + * gsub + * + * Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string). Patterns over 256 characters as written, or with nested quantifiers, are rejected. + */ +export type Html2RssPostProcessorGsub = { + name: 'gsub'; + pattern: string; + replacement: string | { + [key: string]: unknown; + }; + [key: string]: unknown; +}; + +/** + * markdown_to_html + * + * Convert Markdown to HTML (Kramdown) and sanitize the result. Often chained after `template`. + */ +export type Html2RssPostProcessorMarkdownToHtml = { + name: 'markdown_to_html'; + [key: string]: unknown; +}; + +/** + * parse_time + * + * Parse a time string with Time.parse and return RFC822, using the channel `time_zone`. + */ +export type Html2RssPostProcessorParseTime = { + name: 'parse_time'; + [key: string]: unknown; +}; + +/** + * parse_uri + * + * Normalize a URL string; resolve relative URLs against the channel URL. + */ +export type Html2RssPostProcessorParseUri = { + name: 'parse_uri'; + [key: string]: unknown; +}; + +/** + * sanitize_html + * + * Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute URLs, safe link/img attributes, wrap lone images in anchors). + */ +export type Html2RssPostProcessorSanitizeHtml = { + name: 'sanitize_html'; + [key: string]: unknown; +}; + +/** + * substring + * + * Return a slice of the extracted string using Integer `start` and optional `end` (Ruby String#[] range semantics; end may be omitted). + */ +export type Html2RssPostProcessorSubstring = { + end?: number; + name: 'substring'; + start: number; + [key: string]: unknown; +}; + +/** + * template + * + * Format a string with Kernel#format-style placeholders (`%{key}` / `%s`). `%{self}` is the current selector value; other keys resolve sibling selectors. + */ +export type Html2RssPostProcessorTemplate = { + name: 'template'; + string: string; + [key: string]: unknown; +}; + +/** + * One ranked items selector with a sample headline. Best match is first. + */ +export type ItemsSelectorCandidate = { + enhance: boolean; + /** + * Visible text of a matched node + */ + sample: string; + selector: string; +}; + +/** + * One preview meadow row returned by POST /feeds/preview. + */ +export type PreviewSampleItem = { + image?: string | null; + published_at?: string | null; + title: string; + url?: string | null; +}; + +/** + * Ranked selector candidates. Empty buckets are a successful response. + */ +export type SelectorCandidates = { + items: Array; + link: Array; + published: Array; + title: Array; +}; + +/** + * Selectors used to extract article attributes. + */ +export type SelectorsDocument = { + /** + * List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references. + */ + categories?: Array; + /** + * Describes enclosure extraction settings. + */ + enclosure?: { + attribute?: string; + content_type?: string; + /** + * Extractor used to pull a value from the selected element. + */ + extractor?: Html2RssExtractorAttribute | Html2RssExtractorHref | Html2RssExtractorHtml | Html2RssExtractorStatic | Html2RssExtractorText; + /** + * Ordered transforms applied to the extracted value. + */ + post_process?: Array; + static?: string; + }; + /** + * List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references. + */ + guid?: Array; + /** + * Defines the items selector and list-card enhance settings. + */ + items?: { + /** + * List-card enrichment: run Html::ArticleExtractor on each matched item node to fill missing fields from the card HTML. + */ + enhance?: boolean; + order?: 'reverse'; + /** + * Pagination configuration or maximum page count integer. + */ + pagination?: number | { + cursor_path?: string; + increment?: number; + max_pages?: number; + next_url_path?: string; + param?: string; + selector?: string; + start_offset?: number; + start_page?: number; + step?: number; + strategy?: 'rel_next' | 'custom_selector' | 'url_template' | 'offset' | 'json_cursor'; + [key: string]: unknown; + }; + selector?: string; + }; + [key: string]: unknown | { + attribute?: string; + /** + * Extractor used to pull a value from the selected element. + */ + extractor?: Html2RssExtractorAttribute | Html2RssExtractorHref | Html2RssExtractorHtml | Html2RssExtractorStatic | Html2RssExtractorText; + /** + * Ordered transforms applied to the extracted value. + */ + post_process?: Array; + static?: string; + }; +}; + export type GetApiMetadataData = { body?: never; path?: never; @@ -13,7 +237,7 @@ export type GetApiMetadataData = { export type GetApiMetadataResponses = { /** - * returns catalog pointer metadata + * API metadata */ 200: { data: { @@ -31,6 +255,9 @@ export type GetApiMetadataResponses = { access_token_required: boolean; enabled: boolean; }; + studio: { + enabled: boolean; + }; }; }; success: boolean; @@ -48,7 +275,7 @@ export type GetConfigCatalogData = { export type GetConfigCatalogErrors = { /** - * returns 404 when the catalog is disabled + * Config catalog */ 404: { error: string; @@ -59,7 +286,7 @@ export type GetConfigCatalogError = GetConfigCatalogErrors[keyof GetConfigCatalo export type GetConfigCatalogResponses = { /** - * returns the merged catalog with CORS headers + * Config catalog */ 200: { data: { @@ -140,7 +367,7 @@ export type OptionsConfigCatalogData = { export type OptionsConfigCatalogResponses = { /** - * responds to preflight requests with CORS headers + * Config catalog preflight */ 204: void; }; @@ -150,6 +377,7 @@ export type OptionsConfigCatalogResponse = OptionsConfigCatalogResponses[keyof O export type CreateFeedData = { body?: { name?: string; + selectors?: SelectorsDocument; url: string; }; headers: { @@ -162,7 +390,7 @@ export type CreateFeedData = { export type CreateFeedErrors = { /** - * returns 400 when request body exceeds maximum allowed bytes + * Create a feed */ 400: { error: { @@ -176,7 +404,7 @@ export type CreateFeedErrors = { success: boolean; }; /** - * returns 401 with UNAUTHORIZED error payload + * Create a feed */ 401: { error: { @@ -190,7 +418,7 @@ export type CreateFeedErrors = { success: boolean; }; /** - * returns forbidden for authenticated requests when auto source is disabled + * Create a feed */ 403: { error: { @@ -204,7 +432,7 @@ export type CreateFeedErrors = { success: boolean; }; /** - * returns 429 when rate limit is exceeded + * Create a feed */ 429: { error: { @@ -223,7 +451,7 @@ export type CreateFeedError = CreateFeedErrors[keyof CreateFeedErrors]; export type CreateFeedResponses = { /** - * normalizes hostname-only input to https before feed creation + * Create a feed */ 201: { data: { @@ -247,6 +475,170 @@ export type CreateFeedResponses = { export type CreateFeedResponse = CreateFeedResponses[keyof CreateFeedResponses]; +export type PreviewFeedData = { + body?: { + selectors: SelectorsDocument; + url: string; + }; + headers: { + Authorization: string; + }; + path?: never; + query?: never; + url: '/feeds/preview'; +}; + +export type PreviewFeedResponses = { + /** + * Preview a selectors document feed + */ + 200: { + data: { + failure_kind: string | null; + item_count: number; + quality_report: { + metrics: { + short_title_count: number; + }; + warnings: Array; + } | null; + sample_items: Array; + validation_issues: Array; + }; + success: boolean; + }; +}; + +export type PreviewFeedResponse = PreviewFeedResponses[keyof PreviewFeedResponses]; + +export type SuggestSelectorsData = { + body?: { + items_selector: string; + url: string; + }; + headers: { + Authorization: string; + }; + path?: never; + query?: never; + url: '/feeds/suggest_selectors'; +}; + +export type SuggestSelectorsErrors = { + /** + * Suggest selectors for a page + */ + 422: { + error: { + code: string; + kind: string; + message: string; + next_action: string; + retry_action: string; + retryable: boolean; + }; + success: boolean; + }; +}; + +export type SuggestSelectorsError = SuggestSelectorsErrors[keyof SuggestSelectorsErrors]; + +export type SuggestSelectorsResponses = { + /** + * Suggest selectors for a page + */ + 200: { + data: { + admission_drops: { + chrome?: number; + }; + candidates: SelectorCandidates; + segment_strategy: string; + }; + success: boolean; + }; +}; + +export type SuggestSelectorsResponse = SuggestSelectorsResponses[keyof SuggestSelectorsResponses]; + +export type ValidateFeedConfigData = { + body?: { + selectors?: SelectorsDocument; + url?: string; + yaml?: string; + }; + headers: { + Authorization: string; + }; + path?: never; + query?: never; + url: '/feeds/validate'; +}; + +export type ValidateFeedConfigErrors = { + /** + * Validate a selectors document + */ + 400: { + error: { + code: string; + kind: string; + message: string; + next_action: string; + retry_action: string; + retryable: boolean; + }; + success: boolean; + }; + /** + * Validate a selectors document + */ + 401: { + error: { + code: string; + kind: string; + message: string; + next_action: string; + retry_action: string; + retryable: boolean; + }; + success: boolean; + }; +}; + +export type ValidateFeedConfigError = ValidateFeedConfigErrors[keyof ValidateFeedConfigErrors]; + +export type ValidateFeedConfigResponses = { + /** + * Validate a selectors document + */ + 200: { + data: { + report: { + issues: Array<{ + /** + * JSON-ish value, or null when the issue carries none. + */ + actual: unknown; + code: 'constraint' | 'invalid_value' | 'missing_key' | 'parse' | 'type_mismatch' | 'unknown_value'; + /** + * JSON-ish value, or null when the issue carries none. + */ + expected: unknown; + message: string; + path: Array; + }>; + success: boolean; + }; + selectors: null | SelectorsDocument; + yaml: string | null; + }; + success: boolean; + }; +}; + +export type ValidateFeedConfigResponse = ValidateFeedConfigResponses[keyof ValidateFeedConfigResponses]; + export type RenderFeedByTokenData = { body?: never; path: { @@ -258,31 +650,31 @@ export type RenderFeedByTokenData = { export type RenderFeedByTokenErrors = { /** - * returns unauthorized for invalid tokens + * Render feed by token */ 401: string; /** - * returns forbidden when auto source is disabled + * Render feed by token */ 403: string; /** - * returns 422 for empty extraction feeds in xml representation + * Render feed by token */ 422: string; /** - * returns 429 when rate limit is exceeded + * Render feed by token */ 429: string; /** - * returns non-cacheable feed errors when service generation fails + * Render feed by token */ 500: string; /** - * returns 503 when the scraper queue times out + * Render feed by token */ 503: string; /** - * returns 504 when the gateway times out + * Render feed by token */ 504: string; }; @@ -291,7 +683,7 @@ export type RenderFeedByTokenError = RenderFeedByTokenErrors[keyof RenderFeedByT export type RenderFeedByTokenResponses = { /** - * renders feed for a valid token + * Render feed by token */ 200: string; }; @@ -310,7 +702,7 @@ export type GetHealthStatusData = { export type GetHealthStatusErrors = { /** - * returns 401 with UNAUTHORIZED error payload + * Authenticated health check */ 401: { error: { @@ -324,7 +716,7 @@ export type GetHealthStatusErrors = { success: boolean; }; /** - * returns error when configuration fails + * Authenticated health check */ 500: { error: { @@ -343,7 +735,7 @@ export type GetHealthStatusError = GetHealthStatusErrors[keyof GetHealthStatusEr export type GetHealthStatusResponses = { /** - * returns health status when token is valid + * Authenticated health check */ 200: { data: { @@ -372,7 +764,7 @@ export type GetLivenessProbeData = { export type GetLivenessProbeResponses = { /** - * returns liveness status without authentication + * Liveness probe */ 200: { data: { @@ -396,7 +788,7 @@ export type GetReadinessProbeData = { export type GetReadinessProbeResponses = { /** - * returns readiness status without authentication + * Readiness probe */ 200: { data: { @@ -425,7 +817,7 @@ export type ListStrategiesData = { export type ListStrategiesResponses = { /** - * returns available strategies + * List extraction strategies */ 200: { data: { diff --git a/frontend/src/api/http/catalog.ts b/frontend/src/api/http/catalog.ts new file mode 100644 index 000000000..fc95d13ad --- /dev/null +++ b/frontend/src/api/http/catalog.ts @@ -0,0 +1,8 @@ +import { getConfigCatalog } from '../generated/sdk.gen'; +import { jsonBaseUrl, unwrap } from './client'; + +export async function requestConfigCatalog(): Promise { + const result = await unwrap(getConfigCatalog({ baseUrl: jsonBaseUrl() })); + if (!result.ok) return; + return result.body; +} diff --git a/frontend/src/api/http/client.ts b/frontend/src/api/http/client.ts new file mode 100644 index 000000000..f232652cf --- /dev/null +++ b/frontend/src/api/http/client.ts @@ -0,0 +1,51 @@ +export type HttpEnvelope = { + readonly ok: boolean; + readonly status: number; + readonly body: unknown; +}; + +type SdkFields = { + data?: unknown; + error?: unknown; + response?: Response; +}; + +/** True when `error` is an AbortError (DOMException or Error). Neutral HTTP-boundary helper. */ +export function isAbortError(error: unknown): boolean { + return ( + (error instanceof DOMException && error.name === 'AbortError') || + (error instanceof Error && error.name === 'AbortError') + ); +} + +/** Same-origin JSON API prefix. Absolute when `location.origin` exists so `Request` can parse it. */ +export function jsonBaseUrl(): string { + const origin = globalThis.location?.origin; + return origin ? `${origin}/api/v1` : '/api/v1'; +} + +/** + * Maps Hey API `{ data, error, response }` into a closed `{ ok, status, body }` envelope. + * Network failures and missing responses are `ok: false` with status `0`. + */ +export async function unwrap(result: Promise): Promise { + try { + const { data, error, response } = await result; + if (!response) return { ok: false, status: 0, body: error }; + + return { + ok: response.ok, + status: response.status, + body: response.ok ? data : (error ?? data), + }; + } catch (error) { + return { ok: false, status: 0, body: error }; + } +} + +/** Resolve Hey API's Request/URL fetch argument for host and path checks. */ +export function httpRequestUrl(input: RequestInfo | URL): URL { + if (input instanceof URL) return input; + if (typeof input === 'string') return new URL(input, location.origin || 'http://localhost'); + return new URL(input.url); +} diff --git a/frontend/src/api/http/feeds.ts b/frontend/src/api/http/feeds.ts new file mode 100644 index 000000000..bba785ab4 --- /dev/null +++ b/frontend/src/api/http/feeds.ts @@ -0,0 +1,89 @@ +import { client } from '../generated/client.gen'; +import type { + CreateFeedData, + PreviewFeedData, + SuggestSelectorsData, + ValidateFeedConfigData, +} from '../generated/types.gen'; +import type { StudioSelectors } from '../../studio/selectorDraft'; +import { isAbortError, jsonBaseUrl, unwrap, type HttpEnvelope } from './client'; + +type CreateFeedBody = NonNullable; +type PreviewFeedBody = NonNullable; +type ValidateFeedBody = NonNullable; + +const FEED_WRITE_URL = { + create: '/feeds', + preview: '/feeds/preview', + suggest: '/feeds/suggest_selectors', + validate: '/feeds/validate', +} as const satisfies { + create: CreateFeedData['url']; + preview: PreviewFeedData['url']; + suggest: SuggestSelectorsData['url']; + validate: ValidateFeedConfigData['url']; +}; + +function bearerAuth(token: string): string | undefined { + const trimmed = token.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +async function jsonWrite( + url: (typeof FEED_WRITE_URL)[keyof typeof FEED_WRITE_URL], + token: string, + body: unknown, + signal?: AbortSignal +): Promise { + const envelope = await unwrap( + client.post({ + url, + baseUrl: jsonBaseUrl(), + body, + auth: bearerAuth(token), + security: [{ scheme: 'bearer', type: 'http' }], + headers: { 'Content-Type': 'application/json' }, + signal, + }) + ); + if (isAbortError(envelope.body)) throw envelope.body; + return envelope; +} + +/** Single HTTP boundary: narrow studio outbound becomes the wire selectors document. */ +export function postCreateFeed( + url: string, + token: string, + selectors?: StudioSelectors +): Promise { + const body: CreateFeedBody = selectors ? { url, selectors } : { url }; + return jsonWrite(FEED_WRITE_URL.create, token, body); +} + +export function postValidateFeed( + token: string, + url: string, + selectors: StudioSelectors, + signal?: AbortSignal +): Promise { + const body: ValidateFeedBody = { selectors, url }; + return jsonWrite(FEED_WRITE_URL.validate, token, body, signal); +} + +export function postPreviewFeed( + token: string, + url: string, + selectors: StudioSelectors, + signal?: AbortSignal +): Promise { + const body: PreviewFeedBody = { url, selectors }; + return jsonWrite(FEED_WRITE_URL.preview, token, body, signal); +} + +export function postSuggestSelectors( + token: string, + url: string, + signal?: AbortSignal +): Promise { + return jsonWrite(FEED_WRITE_URL.suggest, token, { url }, signal); +} diff --git a/frontend/src/api/http/metadata.ts b/frontend/src/api/http/metadata.ts new file mode 100644 index 000000000..4c1b420c8 --- /dev/null +++ b/frontend/src/api/http/metadata.ts @@ -0,0 +1,17 @@ +import { getApiMetadata } from '../generated/sdk.gen'; +import type { GetApiMetadataResponse } from '../generated/types.gen'; +import { jsonBaseUrl, unwrap } from './client'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +export function isApiMetadataResponse(value: unknown): value is GetApiMetadataResponse { + return isRecord(value) && value.success === true && isRecord(value.data) && isRecord(value.data.instance); +} + +export async function requestApiMetadata(): Promise { + const result = await unwrap(getApiMetadata({ baseUrl: jsonBaseUrl() })); + if (!result.ok || !isApiMetadataResponse(result.body)) return; + return result.body; +} diff --git a/frontend/src/catalog/useCatalogEntries.ts b/frontend/src/catalog/useCatalogEntries.ts index 0f84b236e..3fc3d7cf5 100644 --- a/frontend/src/catalog/useCatalogEntries.ts +++ b/frontend/src/catalog/useCatalogEntries.ts @@ -1,5 +1,6 @@ import { useEffect, useState } from 'preact/hooks'; import type { ApiMetadataRecord } from '../api/contracts'; +import { requestConfigCatalog } from '../api/http/catalog'; import { parseCatalog } from './parseCatalog'; import type { CatalogEntry } from './types'; @@ -15,10 +16,10 @@ const EMPTY: CatalogHookState = { entries: [], starters: [] }; */ export function useCatalogEntries(metadata?: ApiMetadataRecord): CatalogHookState { const [snapshot, setSnapshot] = useState(EMPTY); - const catalog = metadata?.instance.catalog; + const isCatalogEnabled = metadata?.instance.catalog?.enabled === true; useEffect(() => { - if (!catalog?.enabled || !catalog.url) { + if (!isCatalogEnabled) { setSnapshot(EMPTY); return; } @@ -27,13 +28,7 @@ export function useCatalogEntries(metadata?: ApiMetadataRecord): CatalogHookStat const load = async () => { try { - const response = await fetch(catalog.url, { headers: { Accept: 'application/json' } }); - if (!response.ok) { - if (!isCancelled) setSnapshot(EMPTY); - return; - } - - const payload: unknown = await response.json(); + const payload = await requestConfigCatalog(); if (isCancelled) return; const parsed = parseCatalog(payload); setSnapshot({ entries: parsed.entries, starters: parsed.starters }); @@ -46,7 +41,7 @@ export function useCatalogEntries(metadata?: ApiMetadataRecord): CatalogHookStat return () => { isCancelled = true; }; - }, [catalog?.enabled, catalog?.url]); + }, [isCatalogEnabled]); return snapshot; } diff --git a/frontend/src/components/App.tsx b/frontend/src/components/App.tsx index daebf778b..f11635933 100644 --- a/frontend/src/components/App.tsx +++ b/frontend/src/components/App.tsx @@ -43,6 +43,7 @@ export function App() { saveToken, clearToken, feedCreationEnabled, + studioEnabled, mayCreate, } = useSession(); @@ -62,7 +63,9 @@ export function App() { onCancelTokenPrompt, onRetryCreate, onCreateAnother, + generateFromStudio, onRetryPreview, + sourceUrl, setBookmarkletNotice, setTokenDraft, setTokenError, @@ -70,6 +73,7 @@ export function App() { token, isLoading: sessionLoading, feedCreationEnabled, + isStudioEnabled: studioEnabled, mayCreate, saveToken, clearToken, @@ -101,11 +105,20 @@ export function App() { ); } else if (viewModel.kind === 'result') { + const studio = + studioEnabled && token?.trim() && sourceUrl + ? { + token, + url: sourceUrl, + onGenerate: generateFromStudio, + } + : undefined; bodyContent = ( ); } else { diff --git a/frontend/src/components/AppPanels.tsx b/frontend/src/components/AppPanels.tsx index be1963b88..4e02a3d21 100644 --- a/frontend/src/components/AppPanels.tsx +++ b/frontend/src/components/AppPanels.tsx @@ -349,40 +349,31 @@ function TokenGateSection({ ); } -interface ActionFeedbackProperties { - failureMessage: string; - isCreating: boolean; - isShowRetryButton: boolean; - onRetryCreate: () => void; -} - function ActionFeedback({ failureMessage, - isCreating, isShowRetryButton, onRetryCreate, -}: ActionFeedbackProperties) { +}: { + failureMessage: string; + isShowRetryButton: boolean; + onRetryCreate: () => void; +}) { + if (!failureMessage) return; return ( - <> - {failureMessage && ( - - {COPY.tryAgain} - - ) - } - > -

{failureMessage}

- - )} - - {isCreating && } - + + {COPY.tryAgain} + + ) : undefined + } + > +

{failureMessage}

+ ); } @@ -491,7 +482,6 @@ export function CreateFeedPanel({ /> diff --git a/frontend/src/components/Bookmarklet.tsx b/frontend/src/components/Bookmarklet.tsx index 86af64c3d..e8b63ca51 100644 --- a/frontend/src/components/Bookmarklet.tsx +++ b/frontend/src/components/Bookmarklet.tsx @@ -1,21 +1,12 @@ -import { buildAppRouteHref } from '../routes/appRoute'; +import { buildBookmarkletHref } from '../routes/bookmarkletHref'; import { COPY } from '../journey/copy'; export function Bookmarklet({ onClick }: { onClick?: (event: Event) => void }) { - const bookmarkletHref = (() => { - if (globalThis.window === undefined) return '#'; - - const createHref = buildAppRouteHref({ kind: 'create' }); - const targetPrefix = `${createHref}?url=`; - - return `javascript:window.location.assign(${JSON.stringify(targetPrefix)}+encodeURIComponent(window.location.href));`; - })(); - return ( { onClick?.(event); diff --git a/frontend/src/components/PreviewFeedback.tsx b/frontend/src/components/PreviewFeedback.tsx new file mode 100644 index 000000000..c111a16ee --- /dev/null +++ b/frontend/src/components/PreviewFeedback.tsx @@ -0,0 +1,31 @@ +import type { ComponentChildren } from 'preact'; + +interface PreviewFeedbackProperties { + readonly tone: 'loading' | 'error'; + readonly children: ComponentChildren; + readonly onRetry?: () => void; + readonly retryLabel?: string; +} + +/** Shared loading / error / retry chrome for preview and suggestion surfaces. */ +export function PreviewFeedback({ tone, children, onRetry, retryLabel }: PreviewFeedbackProperties) { + if (tone === 'loading') { + return ( +
+
+ ); + } + + return ( +
+ {children} + {onRetry && retryLabel ? ( + + ) : undefined} +
+ ); +} diff --git a/frontend/src/components/PreviewItem.tsx b/frontend/src/components/PreviewItem.tsx index cf5a14e7c..00909f2c9 100644 --- a/frontend/src/components/PreviewItem.tsx +++ b/frontend/src/components/PreviewItem.tsx @@ -1,13 +1,17 @@ +import { useState } from 'preact/hooks'; + interface PreviewItemProperties { title: string; excerpt?: string; url?: string; publishedLabel?: string; + imageUrl?: string; } -export function PreviewItem({ title, excerpt, url, publishedLabel }: PreviewItemProperties) { +export function PreviewItem({ title, excerpt, url, publishedLabel, imageUrl }: PreviewItemProperties) { return ( <> + {imageUrl ? : undefined} {publishedLabel && (

@@ -26,3 +30,19 @@ export function PreviewItem({ title, excerpt, url, publishedLabel }: PreviewItem ); } + +function SampleImage({ url }: { readonly url: string }) { + const [visible, setVisible] = useState(true); + if (!visible) return; + + return ( + setVisible(false)} + /> + ); +} diff --git a/frontend/src/components/ResultDisplay.tsx b/frontend/src/components/ResultDisplay.tsx index 6045e390d..e430bd46b 100644 --- a/frontend/src/components/ResultDisplay.tsx +++ b/frontend/src/components/ResultDisplay.tsx @@ -1,71 +1,122 @@ import type { ComponentChildren } from 'preact'; -import { useEffect, useRef, useState } from 'preact/hooks'; +import { useEffect, useRef } from 'preact/hooks'; +import type { FeedPreviewState, FeedPreviewWarning } from '../api/contracts'; import type { AppViewModel } from '../feed'; +import { useClipboard } from '../hooks/useClipboard'; import { COPY } from '../journey/copy'; +import { ConfigStudio, type AuthenticatedStudioCapability } from '../studio'; import { DominantField } from './DominantField'; +import { Notice } from './Notice'; +import { PreviewFeedback } from './PreviewFeedback'; import { PreviewItem } from './PreviewItem'; interface ResultDisplayProperties { viewModel: Extract; onCreateAnother: () => void; onRetryPreview: () => void; + studio?: AuthenticatedStudioCapability; } -interface PreviewSectionProperties { - ariaLabel: string; - eyebrow?: string; - children: ComponentChildren; +export function ResultDisplay({ + viewModel, + onCreateAnother, + onRetryPreview, + studio, +}: ResultDisplayProperties) { + if (viewModel.phase === 'unresolved') { + return ( + + ); + } + + return ( + + ); } -function PreviewSection({ ariaLabel, eyebrow, children }: PreviewSectionProperties) { +function UnresolvedResult({ + url, + notice, + studio, + onCreateAnother, +}: { + readonly url: string; + readonly notice: string; + readonly studio?: AuthenticatedStudioCapability; + readonly onCreateAnother: () => void; +}) { return ( -

- {eyebrow &&

{eyebrow}

} - {children} +
+
+

{COPY.urlLabel}

+

{url}

+
+ +
+ +

{notice}

+ +
+ + {studio ? ( + + ) : undefined} + +
+ +
); } -export function ResultDisplay({ viewModel, onCreateAnother, onRetryPreview }: ResultDisplayProperties) { - const [copied, setCopied] = useState(false); - const copyResetReference = useRef | undefined>(undefined); +function ReadyResult({ + viewModel, + onCreateAnother, + onRetryPreview, + studio, +}: { + readonly viewModel: Extract; + readonly onCreateAnother: () => void; + readonly onRetryPreview: () => void; + readonly studio?: AuthenticatedStudioCapability; +}) { + const { feedback, copy } = useClipboard(); const copyButtonReference = useRef(null); const { feed, preview, warnings } = viewModel; - - const fullUrl = feed.public_url.startsWith('http') - ? feed.public_url - : `${location.origin}${feed.public_url}`; - const jsonFeedUrl = feed.json_public_url.startsWith('http') - ? feed.json_public_url - : `${location.origin}${feed.json_public_url}`; + const meadow = ; + const fullUrl = absoluteUrl(feed.public_url); + const jsonFeedUrl = absoluteUrl(feed.json_public_url); const subscribeUrl = /^https?:\/\//i.test(fullUrl) ? `feed:${fullUrl}` : undefined; - const canManuallyRetryPreview = - preview.status === 'preview_failed' && warnings.some((warning) => warning.retryable); - const previewMessage = warnings[0]?.message ?? ''; - const hasPreviewItems = preview.items.length > 0; - const isShowPreviewError = - preview.status === 'preview_failed' && !preview.isLoading && !hasPreviewItems && !!previewMessage; useEffect(() => { copyButtonReference.current?.focus(); - return () => { - if (copyResetReference.current) clearTimeout(copyResetReference.current); - }; - }, []); - - const copyToClipboard = async (text: string) => { - try { - await navigator.clipboard.writeText(text); - setCopied(true); - if (copyResetReference.current) clearTimeout(copyResetReference.current); - copyResetReference.current = setTimeout(() => setCopied(false), 2500); - } catch { - // Clipboard may be unavailable in restricted contexts. - } - }; + }, [feed.feed_token]); return ( -
+

{COPY.feedReady}

{feed.name}

@@ -79,9 +130,11 @@ export function ResultDisplay({ viewModel, onCreateAnother, onRetryPreview }: Re readOnly actionRef={copyButtonReference} actionLabel={COPY.copyFeedUrl} - actionText={copied ? COPY.copied : COPY.copy} - actionVariant={copied ? 'soft' : 'default'} - onAction={() => void copyToClipboard(fullUrl)} + actionText={feedback === 'copied' ? COPY.copied : COPY.copy} + actionVariant={feedback === 'copied' ? 'soft' : 'default'} + onAction={() => { + void copy(fullUrl); + }} />
@@ -91,54 +144,94 @@ export function ResultDisplay({ viewModel, onCreateAnother, onRetryPreview }: Re {COPY.openJsonFeed} - {subscribeUrl && ( + {subscribeUrl ? ( {COPY.openInFeedReader} - )} + ) : undefined}
- {preview.isLoading && ( - -
-
- + {studio ? ( + + ) : ( + meadow )} +
+ ); +} - {!preview.isLoading && hasPreviewItems && ( - -
    - {preview.items.map((item) => ( -
  • - -
    • - ))} -
- - )} +function ResultMeadow({ + preview, + warnings, + onRetryPreview, +}: { + readonly preview: FeedPreviewState; + readonly warnings: readonly FeedPreviewWarning[]; + readonly onRetryPreview: () => void; +}) { + const previewMessage = warnings[0]?.message ?? ''; + const hasPreviewItems = preview.items.length > 0; + const isShowError = + preview.status === 'preview_failed' && !preview.isLoading && !hasPreviewItems && !!previewMessage; + const canRetry = isShowError && warnings.some((warning) => warning.retryable); - {isShowPreviewError && ( - -
- {previewMessage} - {canManuallyRetryPreview && ( - - )} -
- - )} + let body: ComponentChildren; + if (preview.isLoading) { + body = {COPY.previewChecking}; + } else if (hasPreviewItems) { + body = ( +
    + {preview.items.map((item) => ( +
  • + +
    • + ))} +
+ ); + } else if (isShowError) { + body = ( + + {previewMessage} + + ); + } else { + return; + } + + return ( +
+ {hasPreviewItems && !preview.isLoading ? ( +

{COPY.previewItemCount(preview.items.length)}

+ ) : undefined} + {body}
); } + +function absoluteUrl(path: string): string { + return path.startsWith('http') ? path : `${location.origin}${path}`; +} diff --git a/frontend/src/feed/decideJourney.ts b/frontend/src/feed/decideJourney.ts index 5c2fedff0..1df597638 100644 --- a/frontend/src/feed/decideJourney.ts +++ b/frontend/src/feed/decideJourney.ts @@ -7,6 +7,12 @@ import type { } from '../api/contracts'; import type { AppRoute } from '../routes/appRoute'; +/** In-memory unresolved result workspace (empty extraction + studio recovery). */ +export interface UnresolvedWorkspace { + readonly url: string; + readonly notice: string; +} + /** Closed UI journey kinds owned by Feed Flow. */ export type AppViewModel = | { kind: 'create' } @@ -14,10 +20,17 @@ export type AppViewModel = | { kind: 'token_prompt'; tokenError: string; error?: FeedCreationError } | { kind: 'result'; + phase: 'ready'; feed: FeedRecord; preview: FeedPreviewState; warnings: FeedPreviewWarning[]; } + | { + kind: 'result'; + phase: 'unresolved'; + url: string; + notice: string; + } | { kind: 'error'; message: string; @@ -37,6 +50,7 @@ export function decideJourney({ route, tokenError, result, + unresolved, }: { creationError?: FeedCreationError; feedFieldErrors: { url: string; form: string }; @@ -44,16 +58,35 @@ export function decideJourney({ route: AppRoute; tokenError: string; result?: CreatedFeedResult; + unresolved?: UnresolvedWorkspace; }): AppViewModel { - if (route.kind === 'result' && result && result.feed.feed_token === route.feedToken) { + // Ready result: keep the panel mounted even when studio re-generate advances + // the token before the hash catches up. + if (route.kind === 'result' && result) { return { kind: 'result', + phase: 'ready', feed: result.feed, preview: result.preview, warnings: result.warnings, }; } + // Unresolved workspace: in-memory only; cold `#/result` without unresolved recovers to create. + if (route.kind === 'result' && unresolved) { + return { + kind: 'result', + phase: 'unresolved', + url: unresolved.url, + notice: unresolved.notice, + }; + } + + // Unmatched / cold result hash without in-memory workspace — Feed Flow recovers to create. + if (route.kind === 'result') { + return { kind: 'create' }; + } + // Token prompt: URL adapter kind and/or in-field token error after Feed Flow navigates. if (route.kind === 'token' || tokenError) { return { kind: 'token_prompt', tokenError, error: creationError }; diff --git a/frontend/src/feed/index.ts b/frontend/src/feed/index.ts index 553e217e3..c4c67e5ce 100644 --- a/frontend/src/feed/index.ts +++ b/frontend/src/feed/index.ts @@ -1,3 +1,3 @@ export { useFeedFlow, type FeedFlowDependencies } from './useFeedFlow'; export { useFeedCreation } from './useFeedCreation'; -export { decideJourney, type AppViewModel } from './decideJourney'; +export { decideJourney, type AppViewModel, type UnresolvedWorkspace } from './decideJourney'; diff --git a/frontend/src/feed/useFeedCreation.ts b/frontend/src/feed/useFeedCreation.ts index 360ac7f57..8778b2648 100644 --- a/frontend/src/feed/useFeedCreation.ts +++ b/frontend/src/feed/useFeedCreation.ts @@ -1,5 +1,7 @@ import { useEffect, useRef, useState } from 'preact/hooks'; -import type { CreatedFeedResult, FeedCreationError } from '../api/contracts'; +import type { CreatedFeedResult, FeedCreationError, PreviewAudience } from '../api/contracts'; +import type { StudioSelectors } from '../studio/selectorDraft'; +import { isAbortError } from '../api/http/client'; import { normalizeFeedCreationError, requestFeedCreation, @@ -8,7 +10,7 @@ import { buildPreviewWarning, loadPreviewItemsWithRetry, PREVIEW_UNAVAILABLE_MESSAGE, - isAbortError, + previewAudienceForToken, } from '../feeds/feedsService'; interface CreationState { @@ -24,6 +26,7 @@ interface CreationState { export function useFeedCreation() { const requestIdReference = useRef(0); const previewAbortControllerReference = useRef(undefined); + const previewAudienceReference = useRef('guest'); const [state, setState] = useState({ isCreating: false }); const cancelPreview = () => { @@ -39,17 +42,28 @@ export function useFeedCreation() { [] ); - async function createFeed(normalizedUrl: string, token: string) { + async function createFeed(normalizedUrl: string, token: string, selectors?: StudioSelectors) { const requestId = requestIdReference.current + 1; requestIdReference.current = requestId; cancelPreview(); setState((previous) => ({ ...previous, isCreating: true, error: undefined })); try { - const feed = await requestFeedCreation(normalizedUrl, token); + const feed = await requestFeedCreation(normalizedUrl, token, selectors); + const audience = previewAudienceForToken(token); + if (requestIdReference.current === requestId) { + previewAudienceReference.current = audience; + } const result = buildCreatedFeedResult(feed); commitResult(result, requestId, setState, requestIdReference); - void hydrateFeedPreview(feed, requestId, setState, requestIdReference, previewAbortControllerReference); + void hydrateFeedPreview( + feed, + audience, + requestId, + setState, + requestIdReference, + previewAbortControllerReference + ); return result; } catch (error) { const structuredError = normalizeFeedCreationError(error); @@ -83,6 +97,7 @@ export function useFeedCreation() { void hydrateFeedPreview( currentResult.feed, + previewAudienceReference.current, requestId, setState, requestIdReference, @@ -103,6 +118,7 @@ export function useFeedCreation() { async function hydrateFeedPreview( feed: CreatedFeedResult['feed'], + audience: PreviewAudience, requestId: number, setState: (value: CreationState | ((previous: CreationState) => CreationState)) => void, requestIdReference: { current: number }, @@ -115,7 +131,7 @@ async function hydrateFeedPreview( commitResult(buildPreviewLoadingResult(feed), requestId, setState, requestIdReference); try { - const previewResult = await loadPreviewItemsWithRetry(feed.json_public_url, controller.signal); + const previewResult = await loadPreviewItemsWithRetry(feed.json_public_url, controller.signal, audience); if (requestIdReference.current !== requestId) return; commitResult( diff --git a/frontend/src/feed/useFeedFlow.ts b/frontend/src/feed/useFeedFlow.ts index 3f0d3bb19..831098ba1 100644 --- a/frontend/src/feed/useFeedFlow.ts +++ b/frontend/src/feed/useFeedFlow.ts @@ -1,14 +1,17 @@ import { useEffect, useRef, useState } from 'preact/hooks'; import { useFeedCreation } from './useFeedCreation'; -import { decideJourney } from './decideJourney'; +import { decideJourney, type UnresolvedWorkspace } from './decideJourney'; import { clearFeedDraftState, loadFeedDraftState, saveFeedDraftState } from '../utils/feedWorkflowStorage'; import { expandCreateUrl } from '../utils/url'; import type { FeedCreationError } from '../api/contracts'; +import type { StudioSelectors } from '../studio/selectorDraft'; +import { isFeedCreationError } from '../feeds/feedsService'; import { COPY } from '../journey/copy'; import type { AppRoute } from '../routes/appRoute'; import type { MayCreateResult } from '../session'; const EMPTY_FEED_ERRORS = { url: '', form: '' }; +const EXTRACTION_EMPTY = 'EXTRACTION_EMPTY'; interface RouteNavigationOptions { replace?: boolean; @@ -18,6 +21,7 @@ export interface FeedFlowDependencies { token: string | undefined; isLoading: boolean; feedCreationEnabled: boolean; + isStudioEnabled: boolean; mayCreate: (accessToken?: string) => MayCreateResult; saveToken: (token: string) => Promise; clearToken: () => void; @@ -33,6 +37,7 @@ export function useFeedFlow({ token, isLoading, feedCreationEnabled, + isStudioEnabled, mayCreate, saveToken, clearToken, @@ -56,8 +61,9 @@ export function useFeedFlow({ const [tokenError, setTokenError] = useState(''); const [bookmarkletNotice, setBookmarkletNotice] = useState(''); const [focusCreateComposerKey, setFocusCreateComposerKey] = useState(0); + const [unresolved, setUnresolved] = useState(); - const routePrefillUrl = route.kind === 'result' ? undefined : route.prefillUrl; + const routePrefillUrl = route.kind === 'create' || route.kind === 'token' ? route.prefillUrl : undefined; const autoSubmitUrlReference = useRef(routePrefillUrl); const hasAutoSubmittedReference = useRef(false); const previousRouteKindReference = useRef(route.kind); @@ -91,6 +97,13 @@ export function useFeedFlow({ clearError(); }; + const openUnresolvedWorkspace = (url: string, notice: string) => { + setUnresolved({ url, notice }); + setFeedFieldErrors(EMPTY_FEED_ERRORS); + clearError(); + navigate({ kind: 'result' }); + }; + const attemptFeedCreation = async (accessToken: string) => { const expanded = expandCreateUrl(feedFormData.url); if ('error' in expanded) { @@ -123,6 +136,7 @@ export function useFeedFlow({ setFeedFormData((previous) => ({ ...previous, url: normalizedUrl })); const createdResult = await createFeed(normalizedUrl, accessToken); clearFeedDraftState(); + setUnresolved(undefined); navigate({ kind: 'result', feedToken: createdResult.feed.feed_token }); setTokenError(''); return true; @@ -139,6 +153,11 @@ export function useFeedFlow({ return false; } + if (canRecoverEmptyExtraction(failure, isStudioEnabled, accessToken)) { + openUnresolvedWorkspace(normalizedUrl, failure.message); + return false; + } + setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, form: failure.message }); if (route.kind === 'token') { // Prevent create-entry auto-submit from clearing Decision error before remount runs. @@ -191,16 +210,23 @@ export function useFeedFlow({ void attemptFeedCreation(token ?? ''); }, [feedFormData.url, isLoading, mayCreate, navigate, route.kind, token]); - // Recover unmatched result routes onto a remounted create view. + // Keep the hash aligned with in-memory result; only bounce empty deep links to create. useEffect(() => { if (route.kind !== 'result') return; - const isMatched = Boolean(result && result.feed.feed_token === route.feedToken); - if (isMatched) return; + if (result) { + if (result.feed.feed_token === route.feedToken) return; + // Studio re-generate commits the new token before navigate runs; follow memory, do not clear. + navigate({ kind: 'result', feedToken: result.feed.feed_token }, { replace: true }); + return; + } + + // Unresolved workspace is in-memory only — cold `#/result` without it recovers to create. + if (!route.feedToken && unresolved) return; // Do not carry a prefill URL — that would re-trigger auto-submit and bounce back to result. navigate({ kind: 'create' }, { replace: true }); - }, [navigate, result, route]); + }, [navigate, result, route, unresolved]); useEffect(() => { const previousKind = previousRouteKindReference.current; @@ -215,6 +241,7 @@ export function useFeedFlow({ if (!didKindChangeToCreate && !isSameKindCreateEntry) return; clearResult(); + setUnresolved(undefined); setTokenError(''); setTokenDraft(''); if (isSameKindCreateEntry) { @@ -231,6 +258,44 @@ export function useFeedFlow({ setFocusCreateComposerKey((current) => current + 1); }, [clearError, clearResult, createEntryKey, feedFieldErrors.form, route]); + const sourceUrl = inMemorySourceUrl(feedFormData.url, result?.feed.url, unresolved?.url); + + const generateFromStudio = async (selectors: StudioSelectors) => { + const expanded = expandCreateUrl(sourceUrl); + if ('error' in expanded) { + throw new Error(expanded.error === 'empty' ? COPY.urlRequired : COPY.invalidUrlFormat); + } + + try { + const createdResult = await createFeed(expanded.ok, token ?? '', selectors); + clearFeedDraftState(); + setUnresolved(undefined); + navigate({ kind: 'result', feedToken: createdResult.feed.feed_token }); + setTokenError(''); + } catch (submitError) { + if (!isFeedCreationError(submitError)) throw submitError; + + if (submitError.kind === 'auth' || submitError.nextAction === 'enter_token') { + clearToken(); + clearError(); + setTokenDraft(''); + if (route.kind !== 'token') navigate({ kind: 'token', prefillUrl: expanded.ok }); + setTokenError(COPY.tokenRejected); + return; + } + + // Keep unresolved workspace anchored when save still returns empty extraction. + if (canRecoverEmptyExtraction(submitError, isStudioEnabled, token ?? '') && unresolved) { + setUnresolved({ url: expanded.ok, notice: submitError.message }); + clearError(); + throw submitError; + } + + clearError(); + throw submitError; + } + }; + const viewModel = decideJourney({ creationError, feedFieldErrors, @@ -238,6 +303,7 @@ export function useFeedFlow({ route, tokenError, result, + unresolved, }); return { @@ -272,12 +338,38 @@ export function useFeedFlow({ }, onCreateAnother: () => { clearResult(); + setUnresolved(undefined); setFocusCreateComposerKey((current) => current + 1); - navigate({ kind: 'create', prefillUrl: feedFormData.url || undefined }); + // Keep the prior URL in the form; omit route prefill so the prefill effect + // does not clear hasAutoSubmitted and auto-resubmit an empty extraction. + hasAutoSubmittedReference.current = true; + navigate({ kind: 'create' }); }, + generateFromStudio, onRetryPreview: retryPreviewFetch, + sourceUrl, setBookmarkletNotice, setTokenDraft, setTokenError, }; } + +function canRecoverEmptyExtraction( + failure: FeedCreationError, + isStudioEnabled: boolean, + accessToken: string +): boolean { + return failure.code === EXTRACTION_EMPTY && isStudioEnabled && accessToken.trim().length > 0; +} + +function inMemorySourceUrl( + formUrl: string, + pageUrl: string | undefined, + unresolvedUrl: string | undefined +): string { + const fromForm = formUrl.trim(); + if (fromForm) return fromForm; + const fromUnresolved = unresolvedUrl?.trim() ?? ''; + if (fromUnresolved) return fromUnresolved; + return pageUrl?.trim() ?? ''; +} diff --git a/frontend/src/feeds/feedParsers.ts b/frontend/src/feeds/feedParsers.ts index 5aaa7f506..023b57ad4 100644 --- a/frontend/src/feeds/feedParsers.ts +++ b/frontend/src/feeds/feedParsers.ts @@ -1,4 +1,14 @@ -import type { FeedPreviewItem } from '../api/contracts'; +import type { FeedPreviewItem, PreviewAudience } from '../api/contracts'; + +// Member cap matches Html2rss::Web::Api::V1::PreviewSamples::SAMPLE_LIMIT. +const PREVIEW_ITEM_LIMIT: Record = { + guest: 5, + member: 10, +}; + +export function previewAudienceForToken(token: string): PreviewAudience { + return token.trim() ? 'member' : 'guest'; +} export function normalizeString(value: unknown): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined; @@ -9,16 +19,22 @@ export function normalizeBoolean(value: unknown, fallback: boolean): boolean { return typeof value === 'boolean' ? value : fallback; } -export function normalizePreviewItems(items: unknown[] | undefined): FeedPreviewItem[] { +export function normalizePreviewItems( + items: unknown[] | undefined, + audience: PreviewAudience = 'guest' +): FeedPreviewItem[] { if (!Array.isArray(items)) return []; return items - .map((item) => normalizePreviewItem(item)) + .map((item) => normalizePreviewItem(item, audience)) .filter((item): item is FeedPreviewItem => item !== undefined) - .slice(0, 5); + .slice(0, PREVIEW_ITEM_LIMIT[audience]); } -export function normalizePreviewItem(value: unknown): FeedPreviewItem | undefined { +export function normalizePreviewItem( + value: unknown, + audience: PreviewAudience = 'guest' +): FeedPreviewItem | undefined { if (!value || typeof value !== 'object') return undefined; const candidate = value as { @@ -34,12 +50,14 @@ export function normalizePreviewItem(value: unknown): FeedPreviewItem | undefine date_modified?: unknown; dateModified?: unknown; url?: unknown; + image?: unknown; }; const title = normalizeString(candidate.title); if (!title) return undefined; const url = normalizeString(candidate.url); + const imageUrl = audience === 'member' ? safePreviewImageUrl(candidate.image) : undefined; return { title, @@ -57,5 +75,14 @@ export function normalizePreviewItem(value: unknown): FeedPreviewItem | undefine candidate.dateModified ) || '', ...(url && { url }), + ...(imageUrl && { imageUrl }), }; } + +function safePreviewImageUrl(value: unknown): string | undefined { + if (typeof value !== 'string') return; + const url = value.trim(); + if (!url.startsWith('https://') && !url.startsWith('http://')) return; + if (/[\s"'<>]/.test(url)) return; + return url; +} diff --git a/frontend/src/feeds/feedPreviewClient.ts b/frontend/src/feeds/feedPreviewClient.ts index 7f25c9df9..9ee0f77c4 100644 --- a/frontend/src/feeds/feedPreviewClient.ts +++ b/frontend/src/feeds/feedPreviewClient.ts @@ -4,7 +4,9 @@ import type { FeedPreviewStatus, FeedPreviewWarning, FeedRecord, + PreviewAudience, } from '../api/contracts'; +import { isAbortError } from '../api/http/client'; import { COPY } from '../journey/copy'; import { buildPreviewWarning } from './feedErrors'; import { normalizePreviewItems } from './feedParsers'; @@ -28,13 +30,6 @@ export function isTransientHttpStatus(status: number): boolean { return [408, 409, 425, 429, 500, 502, 503, 504].includes(status); } -export function isAbortError(error: unknown): boolean { - return ( - (error instanceof DOMException && error.name === 'AbortError') || - (error instanceof Error && error.name === 'AbortError') - ); -} - export async function wait(delayMs: number, signal?: AbortSignal): Promise { if (delayMs <= 0) return; @@ -131,7 +126,8 @@ export function buildPreviewLoadingResult(feed: FeedRecord): CreatedFeedResult { export async function loadPreviewItemsWithRetry( previewUrl: string, - signal?: AbortSignal + signal?: AbortSignal, + audience: PreviewAudience = 'guest' ): Promise { const delays = [0, ...PREVIEW_RETRY_DELAYS_MS]; let latestRetryableFailure: PreviewLoadResult | undefined; @@ -139,7 +135,7 @@ export async function loadPreviewItemsWithRetry( for (const [index, delayMs] of delays.entries()) { if (delayMs > 0) await wait(delayMs, signal); - const result = await loadPreviewItems(previewUrl, signal); + const result = await loadPreviewItems(previewUrl, signal, audience); if (result.status === 'preview_ready') return result; if (result.warnings.every((warning) => !warning.retryable)) return result; @@ -156,7 +152,11 @@ export async function loadPreviewItemsWithRetry( ); } -export async function loadPreviewItems(previewUrl: string, signal?: AbortSignal): Promise { +export async function loadPreviewItems( + previewUrl: string, + signal?: AbortSignal, + audience: PreviewAudience = 'guest' +): Promise { let response: Response; try { @@ -193,7 +193,7 @@ export async function loadPreviewItems(previewUrl: string, signal?: AbortSignal) try { const payload = (await response.json()) as JsonFeedResponse; return { - items: normalizePreviewItems(payload.items), + items: normalizePreviewItems(payload.items, audience), warnings: [], status: 'preview_ready', }; diff --git a/frontend/src/feeds/feedsService.ts b/frontend/src/feeds/feedsService.ts index f207b3851..fee3a92ee 100644 --- a/frontend/src/feeds/feedsService.ts +++ b/frontend/src/feeds/feedsService.ts @@ -1,4 +1,6 @@ import type { FeedRecord } from '../api/contracts'; +import { postCreateFeed } from '../api/http/feeds'; +import type { StudioSelectors } from '../studio/selectorDraft'; import { COPY } from '../journey/copy'; import { buildStructuredError, @@ -6,7 +8,6 @@ import { type RawApiResponse, } from './feedErrors'; import { normalizeString } from './feedParsers'; -import { readJsonResponse } from './feedPreviewClient'; export * from './feedErrors'; export * from './feedParsers'; @@ -23,28 +24,35 @@ interface RawFeedRecord { updated_at?: unknown; } -interface RawFeedPayload { - feed?: RawFeedRecord; +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; } -interface RawFeedApiResponse extends RawApiResponse { - data?: RawFeedPayload; +function feedApiResponse(body: unknown): RawApiResponse | undefined { + if (!isRecord(body)) return; + return { success: body.success, data: body.data, error: body.error }; } -export async function requestFeedCreation(url: string, token: string): Promise { - const response = await fetch(resolveApiUrl('feeds'), { - method: 'POST', - headers: buildCreateHeaders(token), - body: JSON.stringify({ url }), - }); +function rawFeedRecord(body: unknown): RawFeedRecord | undefined { + const payload = feedApiResponse(body); + if (!isRecord(payload?.data)) return; + const feed = payload.data.feed; + return isRecord(feed) ? feed : undefined; +} - const payload = await readJsonResponse(response); +export async function requestFeedCreation( + url: string, + token: string, + selectors?: StudioSelectors +): Promise { + const envelope = await postCreateFeed(url, token, selectors); + const payload = feedApiResponse(envelope.body); - if (!response.ok) { - throw normalizeFeedCreationErrorFromResponse(response.status, payload?.error, payload); + if (!envelope.ok) { + throw normalizeFeedCreationErrorFromResponse(envelope.status, payload?.error, payload); } - const feed = normalizeFeedRecord(payload?.data?.feed); + const feed = normalizeFeedRecord(rawFeedRecord(envelope.body)); if (!feed) { throw buildStructuredError( 'server', @@ -53,7 +61,7 @@ export async function requestFeedCreation(url: string, token: string): Promise = { - Accept: 'application/json', - 'Content-Type': 'application/json', - }; - - if (normalizedToken) { - headers.Authorization = `Bearer ${normalizedToken}`; - } - - return headers; -} diff --git a/frontend/src/hooks/useClipboard.ts b/frontend/src/hooks/useClipboard.ts new file mode 100644 index 000000000..092b7c92f --- /dev/null +++ b/frontend/src/hooks/useClipboard.ts @@ -0,0 +1,34 @@ +import { useEffect, useRef, useState } from 'preact/hooks'; + +/** Closed clipboard UI feedback. Failure is non-destructive: content is unchanged. */ +export type ClipboardFeedback = 'idle' | 'copied' | 'failed'; + +const RESET_MS = 2500; + +/** + * Shared clipboard copy with owned idle/copied/failed feedback and auto-reset. + * Used by feed URL and YAML copy actions. + */ +export function useClipboard() { + const [feedback, setFeedback] = useState('idle'); + const resetReference = useRef | undefined>(undefined); + + useEffect(() => { + return () => { + if (resetReference.current) clearTimeout(resetReference.current); + }; + }, []); + + async function copy(text: string): Promise { + try { + await navigator.clipboard.writeText(text); + setFeedback('copied'); + } catch { + setFeedback('failed'); + } + if (resetReference.current) clearTimeout(resetReference.current); + resetReference.current = setTimeout(() => setFeedback('idle'), RESET_MS); + } + + return { feedback, copy } as const; +} diff --git a/frontend/src/journey/copy.ts b/frontend/src/journey/copy.ts index 7ebe49af6..6f66161ea 100644 --- a/frontend/src/journey/copy.ts +++ b/frontend/src/journey/copy.ts @@ -67,4 +67,32 @@ export const COPY = { footerNav: 'Footer navigation', previewStatus: 'Feed preview status', previewRegion: 'Feed preview', + itemsSelector: 'Items selector', + enhanceItems: 'Enhance items', + /** Ranked selector choices; buckets may be empty. */ + selectorChoices: 'Headline choices', + selectorSuggestionsLoading: 'Finding selector choices', + selectorSuggestionsEmpty: 'No additional selector choices found.', + /** Suggest failure when wire message equals the unresolved Decision notice — never re-echo EXTRACTION_EMPTY. */ + selectorSuggestionsFailed: 'Selector choices unavailable', + retrySelectorSuggestions: 'Try choices again', + choicesFor: (name: string) => `${name} options`, + addSelector: (label: string) => `Add ${label}`, + removeSelector: (label: string) => `Remove ${label}`, + saveAndGenerate: 'Save and generate feed', + /** One-click YAML utility — no expandable preview. */ + copyYaml: 'Copy YAML', + /** Ready-phase handoff — opens a prefilled Feed Directory GitHub issue. */ + proposeDirectory: 'Open Feed Directory issue', + /** Compact one-click activators for dormant optional selector fields. */ + addOptionalFields: 'Add optional selectors', + addOptionalField: (name: string) => `Add ${name}`, + previewFetchedCaption: 'Based on the page already fetched for this URL', + nativeFeedTitle: 'This site already publishes a feed.', + fieldTitle: 'Title', + fieldLink: 'Link', + fieldPublished: 'Published date', + fieldFound: (name: string) => `${name} — found`, + fieldMissing: (name: string) => `${name} — not found`, + selector: 'Selector', } as const; diff --git a/frontend/src/routes/appRoute.ts b/frontend/src/routes/appRoute.ts index 27ec38dbb..5825821f7 100644 --- a/frontend/src/routes/appRoute.ts +++ b/frontend/src/routes/appRoute.ts @@ -11,7 +11,8 @@ export type AppRoute = } | { kind: 'result'; - feedToken: string; + /** Absent on the in-memory unresolved workspace hash `#/result`. */ + feedToken?: string; }; interface RouteNavigationOptions { @@ -27,6 +28,7 @@ interface RouteLocationLike { const ROUTE_PATHS = { create: '/create', token: '/token', + result: '/result', resultPrefix: '/result/', } as const; @@ -43,6 +45,10 @@ export function readAppRoute(locationLike: RouteLocationLike = getCurrentLocatio return prefillUrl ? { kind: 'token', prefillUrl } : { kind: 'token' }; } + if (pathname === ROUTE_PATHS.result) { + return { kind: 'result' }; + } + if (pathname.startsWith(ROUTE_PATHS.resultPrefix)) { const feedToken = pathname.slice(ROUTE_PATHS.resultPrefix.length); if (feedToken) return { kind: 'result', feedToken }; @@ -72,7 +78,7 @@ export function buildAppRouteHref(route: AppRoute, baseHref = getCurrentHref()): return url.href; } - url.hash = `${ROUTE_PATHS.resultPrefix}${route.feedToken}`; + url.hash = route.feedToken ? `${ROUTE_PATHS.resultPrefix}${route.feedToken}` : ROUTE_PATHS.result; url.search = ''; return url.href; } @@ -91,7 +97,12 @@ export function useAppRoute() { const canonicalize = (source: 'mount' | 'navigation') => { const nextRoute = readAppRoute(); - const needsReplace = !location.hash || location.pathname !== '/' || location.hash.startsWith('#!'); + const canonicalHref = buildAppRouteHref(nextRoute); + const needsReplace = + !location.hash || + location.pathname !== '/' || + location.hash.startsWith('#!') || + location.href !== canonicalHref; if (needsReplace) replaceRoute(nextRoute); setRoute(nextRoute); diff --git a/frontend/src/routes/bookmarkletHref.ts b/frontend/src/routes/bookmarkletHref.ts new file mode 100644 index 000000000..f10e76b0b --- /dev/null +++ b/frontend/src/routes/bookmarkletHref.ts @@ -0,0 +1,14 @@ +import { buildAppRouteHref } from './appRoute'; + +/** + * Builds the `javascript:` bookmarklet href that opens create with the current page URL. + * @param baseHref - Origin used for the create route (defaults to the current location). + * @returns Bookmarklet href, or `#` when `window` is unavailable. + */ +export function buildBookmarkletHref(baseHref?: string): string { + if (globalThis.window === undefined && baseHref === undefined) return '#'; + + const createHref = buildAppRouteHref({ kind: 'create' }, baseHref); + const targetPrefix = `${createHref}?url=`; + return `javascript:window.location.assign(${JSON.stringify(targetPrefix)}+encodeURIComponent(window.location.href));`; +} diff --git a/frontend/src/hooks/useApiMetadata.ts b/frontend/src/session/useApiMetadata.ts similarity index 61% rename from frontend/src/hooks/useApiMetadata.ts rename to frontend/src/session/useApiMetadata.ts index cc1e53bba..eb0450330 100644 --- a/frontend/src/hooks/useApiMetadata.ts +++ b/frontend/src/session/useApiMetadata.ts @@ -1,5 +1,6 @@ import { useEffect, useState } from 'preact/hooks'; import type { ApiMetadataRecord } from '../api/contracts'; +import { requestApiMetadata } from '../api/http/metadata'; import { COPY } from '../journey/copy'; interface ApiMetadataState { @@ -8,11 +9,6 @@ interface ApiMetadataState { error?: string; } -interface ApiMetadataPayload { - success?: boolean; - data?: unknown; -} - export function useApiMetadata() { const [state, setState] = useState({ isLoading: true, @@ -25,13 +21,9 @@ export function useApiMetadata() { setState((previous) => ({ ...previous, isLoading: true, error: undefined })); try { - const response = await fetch('/api/v1', { - headers: { Accept: 'application/json' }, - }); - const payload = await parseMetadataPayload(response); - const metadata = payload.data as ApiMetadataRecord | undefined; - - if (!response.ok || !payload.success || !metadata?.instance) { + const payload = await requestApiMetadata(); + const metadata = payload?.data; + if (!metadata?.instance) { throw new Error(COPY.instanceUnavailable); } if (isCancelled) return; @@ -58,14 +50,3 @@ export function useApiMetadata() { return state; } - -async function parseMetadataPayload(response: Response): Promise { - const body = await response.text(); - if (!body.trim()) return {}; - - try { - return JSON.parse(body) as ApiMetadataPayload; - } catch { - return {}; - } -} diff --git a/frontend/src/session/useSession.ts b/frontend/src/session/useSession.ts index 3f0f26287..3bf7db791 100644 --- a/frontend/src/session/useSession.ts +++ b/frontend/src/session/useSession.ts @@ -1,10 +1,11 @@ import { useMemo } from 'preact/hooks'; import type { CatalogEntry } from '../catalog'; import { selectStarterFeeds, useCatalogEntries } from '../catalog'; -import { useApiMetadata } from '../hooks/useApiMetadata'; import { useAccessToken } from './accessToken'; +import { useApiMetadata } from './useApiMetadata'; const DEFAULT_FEED_CREATION = { enabled: true, access_token_required: true }; +const isStudioEnabledByDefault = false; /** Closed gate outcome for attempting feed creation with an optional token. */ export type MayCreateResult = 'proceed' | 'needToken' | 'disabled'; @@ -28,6 +29,7 @@ export function useSession() { const isLoading = tokenLoading || metadataLoading; const feedCreation = metadata?.instance.feed_creation ?? DEFAULT_FEED_CREATION; const feedCreationEnabled = feedCreation.enabled; + const studioEnabled = metadata?.instance.studio?.enabled ?? isStudioEnabledByDefault; const catalog = useCatalogEntries(metadata); const catalogEntries = catalog.entries; const featuredFeeds: CatalogEntry[] = useMemo( @@ -54,6 +56,7 @@ export function useSession() { saveToken, clearToken, feedCreationEnabled, + studioEnabled, mayCreate, }; } diff --git a/frontend/src/studio/ConfigStudio.tsx b/frontend/src/studio/ConfigStudio.tsx new file mode 100644 index 000000000..bad15a8f0 --- /dev/null +++ b/frontend/src/studio/ConfigStudio.tsx @@ -0,0 +1,416 @@ +import type { ComponentChildren, JSX } from 'preact'; +import { useEffect, useRef, useState } from 'preact/hooks'; +import { COPY } from '../journey/copy'; +import { PreviewFeedback } from '../components/PreviewFeedback'; +import { PreviewItem } from '../components/PreviewItem'; +import { Notice } from '../components/Notice'; +import { useClipboard } from '../hooks/useClipboard'; +import { + hasEverySelectorPart, + STUDIO_FIELDS, + type StudioFieldId, + type StudioSelectors, +} from './selectorDraft'; +import { canSaveStudio, studioIssues, studioPreviewSample, useStudio, type StudioState } from './useStudio'; +import { buildDirectoryIssueHref } from './directoryHandoff'; +import type { StudioFieldCandidate, StudioSuggestionState, StudioEditorMode } from './studioModel'; +import { + dormantStudioFields, + groupStudioIssues, + studioFieldSurfaces, + studioSavePresentation, +} from './studioModel'; +import type { StudioSampleItem } from './studioService'; + +/** Authenticated studio session. Absent ⇒ guest / studio-off. */ +export interface AuthenticatedStudioCapability { + readonly token: string; + readonly url: string; + readonly onGenerate: (selectors: StudioSelectors) => Promise; +} + +const FIELD_LABEL: Record = { + title: COPY.fieldTitle, + link: COPY.fieldLink, + published: COPY.fieldPublished, +}; + +interface ConfigStudioProperties { + readonly url: string; + readonly token: string; + readonly mode: StudioEditorMode; + readonly onGenerate: (selectors: StudioSelectors) => Promise; + readonly fallback?: ComponentChildren; + readonly directoryHandoff?: boolean; + /** Unresolved Decision#message — studio must not re-echo. */ + readonly decisionNotice?: string; +} + +export function ConfigStudio(properties: ConfigStudioProperties) { + const { url, token, mode, onGenerate, fallback, directoryHandoff = false, decisionNotice } = properties; + const studio = useStudio({ url, token }); + const { state, suggestion, isDirty } = studio; + const issues = groupStudioIssues(studioIssues(state)); + const [manualFields, setManualFields] = useState>(() => new Set()); + const [focusField, setFocusField] = useState(); + const fieldInputReferences = useRef>>({}); + const { feedback, copy } = useClipboard(); + const sample = studioPreviewSample(state.live); + const sampleItems = sample?.items ?? []; + const isBusy = state.phase === 'validating' || state.phase === 'saving' || state.live.status === 'loading'; + const save = studioSavePresentation(mode, canSaveStudio(state), isDirty); + const surfaces = studioFieldSurfaces(state.draft, suggestion, issues, manualFields); + const dormantFields = dormantStudioFields(surfaces); + const readyCandidates = suggestion.status === 'ready' ? suggestion.candidates : undefined; + + useEffect(() => { + if (focusField === undefined) return; + fieldInputReferences.current[focusField]?.focus(); + setFocusField(undefined); + }, [focusField]); + + const shouldShowCreateMeadow = + fallback !== undefined && sample === undefined && state.live.status !== 'failed'; + const meadow = shouldShowCreateMeadow ? ( + <> + {fallback} + {state.live.status === 'loading' ? ( +
+ {COPY.previewChecking} +
+ ) : undefined} + + ) : ( + + ); + + return ( +
+
+ {state.phase === 'failed' && !isSameNotice(state.message, decisionNotice) ? ( + + {state.message.trim() && state.message !== COPY.createFailedTitle ? ( +

{state.message}

+ ) : undefined} + + ) : undefined} + +
+ +
+ +
+
+ issue.message)} + onInput={(value) => studio.setItemsSelector(value)} + /> + +
+ + {STUDIO_FIELDS.map((field) => { + if (surfaces[field].status !== 'visible') return; + const label = FIELD_LABEL[field]; + return ( +
+ { + fieldInputReferences.current[field] = element; + }} + errors={issues.fields[field].map((issue) => issue.message)} + onInput={(value) => studio.setFieldSelector(field, value)} + /> + studio.toggleFieldSelector(field, selector)} + /> +
+ ); + })} + + {dormantFields.length > 0 ? ( +
+ {dormantFields.map((field) => ( + + ))} +
+ ) : undefined} +
+ +
+ + issue.message)} /> +
+ + {issues.hidden.length > 0 ? ( + + {issues.hidden.map((issue) => ( +

{issue.message}

+ ))} + + ) : undefined} + + {sample?.nativeFeedUrl ? ( + + + {sample.nativeFeedUrl} + + + ) : undefined} + +
+ + + {directoryHandoff ? ( + + {COPY.proposeDirectory} + + ) : undefined} +
+
+ + {meadow} +
+ ); +} + +function SuggestionStatus({ + suggestion, + decisionNotice, + onRetry, +}: { + readonly suggestion: StudioSuggestionState; + readonly decisionNotice?: string; + readonly onRetry: () => void; +}) { + if (suggestion.status === 'loading') { + return {COPY.selectorSuggestionsLoading}; + } + if (suggestion.status === 'empty') { + return

{COPY.selectorSuggestionsEmpty}

; + } + if (suggestion.status === 'failed') { + const message = isSameNotice(suggestion.message, decisionNotice) + ? COPY.selectorSuggestionsFailed + : suggestion.message; + return ( + + {message} + + ); + } + return; +} + +function isSameNotice(message: string, decisionNotice: string | undefined): boolean { + return decisionNotice !== undefined && message.trim() === decisionNotice.trim(); +} + +function FieldBlock({ + id, + label, + ariaLabel, + value, + errors, + inputReference, + onInput, +}: { + readonly id: string; + readonly label: string; + readonly ariaLabel?: string; + readonly value: string; + readonly errors: readonly string[]; + readonly inputReference?: (element: HTMLInputElement | null) => void; + readonly onInput: (value: string) => void; +}) { + return ( + + ); +} + +function ChoiceChips({ + label, + current, + choices, + onToggle, +}: { + readonly label: string; + readonly current: string; + readonly choices: readonly StudioFieldCandidate[]; + readonly onToggle: (selector: string) => void; +}) { + if (choices.length === 0) return; + return ( +
    + {choices.map((choice) => { + const pressed = hasEverySelectorPart(current, choice.selector); + return ( +
  • + +
    • + ); + })} +
+ ); +} + +function PreviewPane({ + state, + token, + onRetry, +}: { + readonly state: StudioState; + readonly token: string; + readonly onRetry: () => void; +}) { + const items = studioPreviewSample(state.live)?.items ?? []; + const canShowImages = token.trim().length > 0; + + return ( +
+ {state.live.status === 'loading' ? ( + {COPY.previewChecking} + ) : undefined} + {state.live.status === 'failed' ? ( + + {state.live.message} + + ) : undefined} + {items.length > 0 ? ( + <> +

{COPY.previewFetchedCaption}

+

{COPY.previewItemCount(items.length)}

+
    + {items.map((item, index) => ( +
  • + + {item.url ?

    {item.url}

    : undefined} +
    • + ))} +
+ + ) : undefined} +
+ ); +} + +function FieldErrors({ messages }: { readonly messages: readonly string[] }) { + if (messages.length === 0) return; + return ( + <> + {messages.map((message, index) => ( + + {message} + + ))} + + ); +} + +function fieldStatusLabel( + label: string, + sample: ReturnType, + sampleItems: readonly StudioSampleItem[], + field: StudioFieldId +): string { + if (sample === undefined) return label; + return hasSampleField(sampleItems, field) ? COPY.fieldFound(label) : COPY.fieldMissing(label); +} + +function hasSampleField(items: readonly StudioSampleItem[], field: StudioFieldId): boolean { + switch (field) { + case 'title': { + return items.some((item) => item.title.trim().length > 0); + } + case 'link': { + return items.some((item) => item.url !== undefined); + } + case 'published': { + return items.some((item) => item.publishedAt !== undefined); + } + } +} diff --git a/frontend/src/studio/directoryHandoff.ts b/frontend/src/studio/directoryHandoff.ts new file mode 100644 index 000000000..6342de828 --- /dev/null +++ b/frontend/src/studio/directoryHandoff.ts @@ -0,0 +1,21 @@ +/** GitHub new-issue handoff for Riley — not in-app directory authoring. */ + +const DIRECTORY_ISSUE_NEW = 'https://github.com/html2rss/html2rss-configs/issues/new'; + +/** + * Build a prefilled html2rss-configs issue URL. + * Body is channel.url + studio selectors YAML plus blank directory fields. + * Never includes access or feed tokens. + */ +export function buildDirectoryIssueHref(channelUrl: string, selectorsYaml: string): string { + const title = `Add feed: ${channelUrl}`; + const body = buildDirectoryIssueBody(channelUrl, selectorsYaml); + const parameters = new URLSearchParams({ title, body }); + return `${DIRECTORY_ISSUE_NEW}?${parameters.toString()}`; +} + +/** YAML stub for the issue body — blank directory.title / directory.topics for Riley. */ +export function buildDirectoryIssueBody(channelUrl: string, selectorsYaml: string): string { + const base = selectorsYaml.trim() || `channel:\n url: ${channelUrl}\nselectors: {}\n`; + return `${base.trimEnd()}\n\ndirectory:\n title:\n topics:\n`; +} diff --git a/frontend/src/studio/index.ts b/frontend/src/studio/index.ts new file mode 100644 index 000000000..340bcc017 --- /dev/null +++ b/frontend/src/studio/index.ts @@ -0,0 +1,7 @@ +/** + * Narrow studio public facade: editor, capability/mode types, outbound selectors. + * Internal model/service/hook modules stay importable for tests and adjacent owners. + */ +export { ConfigStudio, type AuthenticatedStudioCapability } from './ConfigStudio'; +export type { StudioEditorMode } from './studioModel'; +export type { StudioSelectors } from './selectorDraft'; diff --git a/frontend/src/studio/selectorDraft.ts b/frontend/src/studio/selectorDraft.ts new file mode 100644 index 000000000..35b6b9fe1 --- /dev/null +++ b/frontend/src/studio/selectorDraft.ts @@ -0,0 +1,272 @@ +/** + * Closed studio draft for selector authoring. + * {@link selectorsFromDraft} yields the studio subset; HTTP widens at one boundary. + * Hydrate fails closed on keys outside the subset. + */ + +export const STUDIO_FIELDS = ['title', 'link', 'published'] as const; +export type StudioFieldId = (typeof STUDIO_FIELDS)[number]; + +const STUDIO_FIELD_WIRE = [ + { field: 'title', key: 'title', extractor: 'text' }, + { field: 'link', key: 'url', extractor: 'href' }, + { field: 'published', key: 'published_at', extractor: 'text' }, +] as const satisfies readonly { + field: StudioFieldId; + key: 'title' | 'url' | 'published_at'; + extractor: 'text' | 'href'; +}[]; + +/** Root keys {@link draftFromSelectors} accepts. */ +const STUDIO_ROOT_KEYS = new Set(['items', ...STUDIO_FIELD_WIRE.map(({ key }) => key)]); + +/** Items keys the studio draft owns. */ +const STUDIO_ITEMS_KEYS = new Set(['selector', 'enhance']); + +export interface FieldDraft { + readonly selector: string; +} + +export interface StudioDraft { + readonly itemsSelector: string; + readonly enhance: boolean; + readonly fields: Readonly>; +} + +/** Narrow outbound selectors the studio authors. */ +export type StudioSelectors = { + readonly items: { + readonly selector: string; + /** Always sent — omitting `false` lets the gem default `enhance: true`. */ + readonly enhance: boolean; + }; + readonly title?: { + readonly selector: string; + readonly extractor: 'text'; + }; + readonly url?: { + readonly selector: string; + readonly extractor: 'href'; + }; + readonly published_at?: { + readonly selector: string; + readonly extractor: 'text'; + }; +}; + +type StudioSelectorsBuilder = { + items: StudioSelectors['items']; + title?: StudioSelectors['title']; + url?: StudioSelectors['url']; + published_at?: StudioSelectors['published_at']; +}; + +export type IssuePlace = + | { readonly kind: 'items' } + | { readonly kind: 'enhance' } + | { readonly kind: 'field'; readonly field: StudioFieldId; readonly control: 'selector' } + | { readonly kind: 'hidden' }; + +/** Wire spelling of Html2rss::Selectors::DEFAULT_ITEMS_SELECTOR. */ +export const DEFAULT_ITEMS_SELECTOR = 'a[href]'; + +export function isPlainRecord(value: unknown): value is Record { + return ( + typeof value === 'object' && value instanceof Object && Object.getPrototypeOf(value) === Object.prototype + ); +} + +export function emptyDraft(): StudioDraft { + return { + itemsSelector: DEFAULT_ITEMS_SELECTOR, + enhance: false, + fields: emptyFields(), + }; +} + +export function selectorsFromDraft(draft: StudioDraft): StudioSelectors { + const selectors: StudioSelectorsBuilder = { + items: { selector: draft.itemsSelector, enhance: draft.enhance }, + }; + + if (draft.fields.title.selector.length > 0) { + selectors.title = { selector: draft.fields.title.selector, extractor: 'text' }; + } + if (draft.fields.link.selector.length > 0) { + selectors.url = { selector: draft.fields.link.selector, extractor: 'href' }; + } + if (draft.fields.published.selector.length > 0) { + selectors.published_at = { selector: draft.fields.published.selector, extractor: 'text' }; + } + + return selectors; +} + +export function draftFromSelectors(value: unknown): StudioDraft | undefined { + if (!isPlainRecord(value)) return; + + for (const key of Object.keys(value)) { + if (!STUDIO_ROOT_KEYS.has(key)) return; + } + + let items: Record | undefined; + if (Object.hasOwn(value, 'items')) { + if (!isPlainRecord(value.items)) return; + for (const key of Object.keys(value.items)) { + if (!STUDIO_ITEMS_KEYS.has(key)) return; + } + items = value.items; + } + + const fields = emptyFields(); + for (const mapping of STUDIO_FIELD_WIRE) { + const { key } = mapping; + if (!Object.hasOwn(value, key)) continue; + const parsed = parseField(mapping, value[key]); + if (!parsed) return; + fields[parsed.field] = parsed.draft; + } + + return { + itemsSelector: typeof items?.selector === 'string' ? items.selector : '', + enhance: items?.enhance === true, + fields, + }; +} + +export function withItemsSelector(draft: StudioDraft, itemsSelector: string): StudioDraft { + return { ...draft, itemsSelector }; +} + +export function withEnhance(draft: StudioDraft, isEnhanced: boolean): StudioDraft { + return { ...draft, enhance: isEnhanced }; +} + +export function withFieldSelector(draft: StudioDraft, field: StudioFieldId, selector: string): StudioDraft { + return { + ...draft, + fields: { ...draft.fields, [field]: { selector } }, + }; +} + +/** + * Selector parts split on commas outside `()`, `[]`, and quotes. + */ +export function splitSelectorParts(selector: string): readonly string[] { + const parts: string[] = []; + let start = 0; + let quote: '"' | "'" | undefined; + let paren = 0; + let bracket = 0; + + for (let index = 0; index < selector.length; index += 1) { + const char = selector[index]; + if (char === '\\') { + index += 1; + continue; + } + if (quote) { + if (char === quote) quote = undefined; + continue; + } + if (char === '"' || char === "'") { + quote = char; + continue; + } + if (char === '(') { + paren += 1; + continue; + } + if (char === ')' && paren > 0) { + paren -= 1; + continue; + } + if (char === '[') { + bracket += 1; + continue; + } + if (char === ']' && bracket > 0) { + bracket -= 1; + continue; + } + if (char === ',' && paren === 0 && bracket === 0) { + pushPart(parts, selector.slice(start, index)); + start = index + 1; + } + } + + pushPart(parts, selector.slice(start)); + return parts; +} + +function joinParts(parts: readonly string[]): string { + return parts.join(', '); +} + +/** True when every part of `candidate` is already an exact part of `current`. */ +export function hasEverySelectorPart(current: string, candidate: string): boolean { + const wanted = splitSelectorParts(candidate); + if (wanted.length === 0) return false; + const present = splitSelectorParts(current); + return wanted.every((part) => present.includes(part)); +} + +/** Add missing candidate parts, or remove them when all are present. */ +export function toggleSelectorParts(current: string, candidate: string): string { + const present = splitSelectorParts(current); + const incoming = splitSelectorParts(candidate); + if (incoming.length === 0) return joinParts(present); + if (incoming.every((part) => present.includes(part))) { + const remove = new Set(incoming); + return joinParts(present.filter((part) => !remove.has(part))); + } + + const next = [...present]; + for (const part of incoming) { + if (!next.includes(part)) next.push(part); + } + return joinParts(next); +} + +export function issuePlace(path: readonly string[]): IssuePlace { + const scope = selectorScope(path); + const head = scope[0]; + if (head === 'items') { + return scope.includes('enhance') ? { kind: 'enhance' } : { kind: 'items' }; + } + + const field = + typeof head === 'string' ? STUDIO_FIELD_WIRE.find(({ key }) => key === head)?.field : undefined; + if (!field) return { kind: 'hidden' }; + return { kind: 'field', field, control: 'selector' }; +} + +function pushPart(parts: string[], raw: string) { + const trimmed = raw.trim(); + if (trimmed.length > 0) parts.push(trimmed); +} + +function emptyFields(): Record { + return { + title: { selector: '' }, + link: { selector: '' }, + published: { selector: '' }, + }; +} + +function parseField( + mapping: (typeof STUDIO_FIELD_WIRE)[number], + value: unknown +): { readonly field: StudioFieldId; readonly draft: FieldDraft } | undefined { + if (!isPlainRecord(value) || typeof value.selector !== 'string') return; + if (value.extractor !== undefined && value.extractor !== mapping.extractor) return; + for (const nested of Object.keys(value)) { + if (nested !== 'selector' && nested !== 'extractor') return; + } + return { field: mapping.field, draft: { selector: value.selector } }; +} + +function selectorScope(path: readonly string[]): readonly string[] { + const index = path.indexOf('selectors'); + return index === -1 ? path : path.slice(index + 1); +} diff --git a/frontend/src/studio/studioModel.ts b/frontend/src/studio/studioModel.ts new file mode 100644 index 000000000..48c3e4ede --- /dev/null +++ b/frontend/src/studio/studioModel.ts @@ -0,0 +1,157 @@ +import { issuePlace, STUDIO_FIELDS, type StudioDraft, type StudioFieldId } from './selectorDraft'; + +export type StudioEditorMode = 'ready' | 'unresolved'; + +export interface StudioIssue { + readonly path: readonly string[]; + readonly code: string; + readonly message: string; +} + +/** One ranked selector candidate. Best match is first in its bucket. */ +export interface StudioFieldCandidate { + readonly selector: string; + readonly sample: string; +} +export interface StudioCandidates { + readonly items: readonly StudioFieldCandidate[]; + readonly fields: Readonly>; +} + +export type StudioSuggestionResult = + { readonly status: 'empty' } | { readonly status: 'ready'; readonly candidates: StudioCandidates }; + +export type StudioSuggestionState = + | StudioSuggestionResult + | { readonly status: 'loading' } + | { readonly status: 'failed'; readonly message: string }; + +export interface GroupedStudioIssues { + readonly items: readonly StudioIssue[]; + readonly enhance: readonly StudioIssue[]; + readonly fields: Readonly>; + readonly hidden: readonly StudioIssue[]; +} + +export type OptionalFieldEvidence = 'selector' | 'candidate' | 'issue' | 'manual'; + +export type OptionalFieldSurface = + | { readonly status: 'dormant' } + | { readonly status: 'visible'; readonly evidence: readonly OptionalFieldEvidence[] }; + +export type StudioFieldSurfaces = Readonly>; + +export interface StudioSavePresentation { + readonly enabled: boolean; + readonly className: 'btn btn--primary' | 'btn btn--ghost'; +} + +export type StudioPreviewFailure = + | { readonly kind: 'known'; readonly code: 'execution' | 'min_items' } + | { readonly kind: 'unknown'; readonly code: string }; + +export function suggestionResult(candidates: StudioCandidates): StudioSuggestionResult { + if ( + candidates.items.length === 0 && + STUDIO_FIELDS.every((field) => candidates.fields[field].length === 0) + ) { + return { status: 'empty' }; + } + return { status: 'ready', candidates }; +} + +export function groupStudioIssues(issues: readonly StudioIssue[]): GroupedStudioIssues { + const items: StudioIssue[] = []; + const enhance: StudioIssue[] = []; + const fields: Record = { + title: [], + link: [], + published: [], + }; + const hidden: StudioIssue[] = []; + + for (const issue of issues) { + addGroupedIssue(issue, { items, enhance, fields, hidden }); + } + + return { items, enhance, fields, hidden }; +} + +export function studioFieldSurfaces( + draft: StudioDraft, + suggestion: StudioSuggestionState, + issues: GroupedStudioIssues, + manualFields: ReadonlySet = new Set() +): StudioFieldSurfaces { + const surfaces: Record = { + title: { status: 'dormant' }, + link: { status: 'dormant' }, + published: { status: 'dormant' }, + }; + + for (const field of STUDIO_FIELDS) { + const evidence: OptionalFieldEvidence[] = []; + if (draft.fields[field].selector.trim()) evidence.push('selector'); + if (suggestion.status === 'ready' && suggestion.candidates.fields[field].length > 0) { + evidence.push('candidate'); + } + if (issues.fields[field].length > 0) evidence.push('issue'); + if (manualFields.has(field)) evidence.push('manual'); + surfaces[field] = evidence.length === 0 ? { status: 'dormant' } : { status: 'visible', evidence }; + } + + return surfaces; +} + +/** Optional fields still waiting on selector, candidate, issue, or manual evidence. */ +export function dormantStudioFields(surfaces: StudioFieldSurfaces): readonly StudioFieldId[] { + return STUDIO_FIELDS.filter((field) => surfaces[field].status === 'dormant'); +} + +export function studioSavePresentation( + mode: StudioEditorMode, + canSave: boolean, + isDirty: boolean +): StudioSavePresentation { + const isPrimary = mode === 'unresolved' || isDirty; + return { + enabled: canSave && isPrimary, + className: isPrimary ? 'btn btn--primary' : 'btn btn--ghost', + }; +} + +export function normalizePreviewFailure(value: string | undefined): StudioPreviewFailure | undefined { + const code = value?.trim(); + if (!code) return; + if (code === 'execution' || code === 'min_items') return { kind: 'known', code }; + return { kind: 'unknown', code }; +} + +function addGroupedIssue( + issue: StudioIssue, + groups: { + items: StudioIssue[]; + enhance: StudioIssue[]; + fields: Record; + hidden: StudioIssue[]; + } +) { + const place = issuePlace(issue.path); + switch (place.kind) { + case 'items': { + groups.items.push(issue); + return; + } + case 'enhance': { + groups.enhance.push(issue); + return; + } + case 'field': { + groups.fields[place.field].push(issue); + return; + } + case 'hidden': { + groups.hidden.push(issue); + } + } +} diff --git a/frontend/src/studio/studioService.ts b/frontend/src/studio/studioService.ts new file mode 100644 index 000000000..4db96e36e --- /dev/null +++ b/frontend/src/studio/studioService.ts @@ -0,0 +1,271 @@ +/** + * Studio transport — sole unwrap of `{ success, data }`. + */ +import type { ValidateFeedConfigResponses } from '../api/generated'; +import { isAbortError, type HttpEnvelope } from '../api/http/client'; +import { postPreviewFeed, postSuggestSelectors, postValidateFeed } from '../api/http/feeds'; +import { COPY } from '../journey/copy'; +import { + draftFromSelectors, + isPlainRecord, + selectorsFromDraft, + type StudioDraft, + type StudioSelectors, +} from './selectorDraft'; +import { + normalizePreviewFailure, + suggestionResult, + type StudioCandidates, + type StudioIssue, + type StudioPreviewFailure, + type StudioSuggestionResult, +} from './studioModel'; + +type GeneratedIssue = ValidateFeedConfigResponses[200]['data']['report']['issues'][number]; +export type StudioIssueCode = GeneratedIssue['code']; + +export interface StudioValidationOutcome { + readonly valid: boolean; + readonly issues: readonly StudioIssue[]; + readonly draft: StudioDraft | undefined; + readonly yaml: string | undefined; +} + +export interface StudioSampleItem { + readonly title: string; + readonly url: string | undefined; + readonly publishedAt: string | undefined; + readonly imageUrl: string | undefined; +} + +export interface StudioPreview { + readonly items: readonly StudioSampleItem[]; + readonly nativeFeedUrl: string | undefined; + readonly failure: StudioPreviewFailure | undefined; +} + +type StudioPayloadField = + | 'report' + | 'selectors' + | 'yaml' + | 'issues' + | 'sample_items' + | 'quality_report' + | 'failure_kind' + | 'candidates'; + +export class StudioRequestError extends Error { + readonly field: StudioPayloadField | undefined; + + constructor(message: string, field?: StudioPayloadField) { + super(message); + this.name = 'StudioRequestError'; + this.field = field; + } +} + +function invalidStudioField(field: StudioPayloadField): never { + throw new StudioRequestError(COPY.unableToCompleteCreation, field); +} + +function expectRecord(value: unknown, field: StudioPayloadField): Record { + if (!isPlainRecord(value)) invalidStudioField(field); + return value; +} + +function expectString(value: unknown, field: StudioPayloadField): string { + if (typeof value !== 'string') invalidStudioField(field); + return value; +} + +function expectArray(value: unknown, field: StudioPayloadField): readonly unknown[] { + if (!Array.isArray(value)) invalidStudioField(field); + return value; +} + +function listedIssueCodes( + codes: [StudioIssueCode] extends [T[number]] ? T : never +): T { + return codes; +} + +const STUDIO_ISSUE_CODES = listedIssueCodes([ + 'constraint', + 'invalid_value', + 'missing_key', + 'parse', + 'type_mismatch', + 'unknown_value', +]); + +export function validateStudioSelectors( + token: string, + url: string, + draft: StudioDraft, + signal?: AbortSignal +): Promise { + return validateStudio(token, url, selectorsFromDraft(draft), signal); +} + +export async function previewStudioFeed( + token: string, + url: string, + draft: StudioDraft, + signal?: AbortSignal +): Promise { + const data = await studioData(await postPreviewFeed(token, url, selectorsFromDraft(draft), signal)); + return parsePreview(data); +} + +export async function suggestStudioSelectors( + token: string, + url: string, + signal?: AbortSignal +): Promise { + const data = await studioData(await postSuggestSelectors(token, url, signal)); + return parseSuggestion(data); +} + +async function validateStudio( + token: string, + url: string, + selectors: StudioSelectors, + signal?: AbortSignal +): Promise { + return parseValidation(await studioData(await postValidateFeed(token, url, selectors, signal))); +} + +async function studioData(envelope: HttpEnvelope): Promise { + const payload = envelope.body; + if (isAbortError(payload)) throw payload; + if (!envelope.ok || !isPlainRecord(payload) || payload.success !== true) { + throw new StudioRequestError(errorMessage(payload) ?? COPY.unableToCompleteCreation); + } + + return payload.data; +} + +function errorMessage(payload: unknown): string | undefined { + if (!isPlainRecord(payload) || !isPlainRecord(payload.error)) return; + const message = payload.error.message; + return typeof message === 'string' && message.trim() ? message : undefined; +} + +function parseValidation(data: unknown): StudioValidationOutcome { + const root = expectRecord(data, 'report'); + const report = expectRecord(root.report, 'report'); + if (typeof report.success !== 'boolean') invalidStudioField('report'); + + return { + valid: report.success, + issues: expectArray(report.issues, 'issues').map((item) => parseIssue(item)), + draft: parseSelectorsField(root.selectors), + yaml: parseYamlField(root.yaml), + }; +} + +function parseSelectorsField(value: unknown): StudioDraft | undefined { + if (isAbsent(value)) return; + const draft = draftFromSelectors(value); + if (!draft) invalidStudioField('selectors'); + return draft; +} + +function parseYamlField(value: unknown): string | undefined { + if (isAbsent(value)) return; + return expectString(value, 'yaml'); +} + +function parseIssue(value: unknown): StudioIssue { + const record = expectRecord(value, 'issues'); + const message = expectString(record.message, 'issues'); + const code = knownIssueCode(record.code); + if (!code) invalidStudioField('issues'); + + const path = Array.from(expectArray(record.path, 'issues'), (segment) => expectString(segment, 'issues')); + return { path, code, message }; +} + +function knownIssueCode(value: unknown): StudioIssueCode | undefined { + if (typeof value !== 'string') return; + return STUDIO_ISSUE_CODES.find((code) => code === value); +} + +function parsePreview(data: unknown): StudioPreview { + const root = expectRecord(data, 'sample_items'); + if (typeof root.item_count !== 'number') invalidStudioField('sample_items'); + + return { + items: expectArray(root.sample_items, 'sample_items').map((item) => parseSampleItem(item)), + nativeFeedUrl: parseNativeFeed(root.quality_report), + failure: parseFailureKind(root.failure_kind), + }; +} + +function parseSampleItem(value: unknown): StudioSampleItem { + const record = expectRecord(value, 'sample_items'); + return { + title: expectString(record.title, 'sample_items'), + url: optionalText(record.url, 'sample_items'), + publishedAt: optionalText(record.published_at, 'sample_items'), + imageUrl: optionalImageUrl(record.image), + }; +} + +function parseNativeFeed(value: unknown): string | undefined { + if (isAbsent(value)) return; + const report = expectRecord(value, 'quality_report'); + if (!Object.hasOwn(report, 'native_feed') || isAbsent(report.native_feed)) return; + return optionalText(report.native_feed, 'quality_report'); +} + +function optionalImageUrl(value: unknown): string | undefined { + if (typeof value !== 'string') return; + const trimmed = value.trim(); + if (trimmed.startsWith('https://') || trimmed.startsWith('http://')) return trimmed; + return; +} + +function optionalText(value: unknown, field: StudioPayloadField): string | undefined { + if (isAbsent(value)) return; + const trimmed = expectString(value, field).trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function parseFailureKind(value: unknown): StudioPreviewFailure | undefined { + if (isAbsent(value)) return; + return normalizePreviewFailure(expectString(value, 'failure_kind')); +} + +function parseSuggestion(data: unknown): StudioSuggestionResult { + const root = expectRecord(data, 'candidates'); + const buckets = expectRecord(root.candidates, 'candidates'); + const candidates: StudioCandidates = { + items: parseCandidates(buckets.items, true), + fields: { + title: parseCandidates(buckets.title, false), + link: parseCandidates(buckets.link, false), + published: parseCandidates(buckets.published, false), + }, + }; + return suggestionResult(candidates); +} + +function parseCandidates(value: unknown, isItems: boolean): StudioCandidates['items'] { + return expectArray(value, 'candidates').map((entry) => parseCandidate(entry, isItems)); +} + +function parseCandidate(value: unknown, isItems: boolean): StudioCandidates['items'][number] { + const record = expectRecord(value, 'candidates'); + const allowedKeys = isItems ? new Set(['selector', 'enhance', 'sample']) : new Set(['selector', 'sample']); + if (Object.keys(record).some((key) => !allowedKeys.has(key))) invalidStudioField('candidates'); + const selector = expectString(record.selector, 'candidates').trim(); + const sample = expectString(record.sample, 'candidates').trim(); + if (isItems && typeof record.enhance !== 'boolean') invalidStudioField('candidates'); + if (selector.length === 0 || sample.length === 0) invalidStudioField('candidates'); + return { selector, sample }; +} + +function isAbsent(value: unknown): boolean { + return value === undefined || value === null; +} diff --git a/frontend/src/studio/useStudio.ts b/frontend/src/studio/useStudio.ts new file mode 100644 index 000000000..3af671d34 --- /dev/null +++ b/frontend/src/studio/useStudio.ts @@ -0,0 +1,355 @@ +import { useEffect, useRef, useState } from 'preact/hooks'; +import { isAbortError } from '../api/http/client'; +import { isFeedCreationError } from '../feeds/feedsService'; +import { COPY } from '../journey/copy'; +import { + previewStudioFeed, + StudioRequestError, + suggestStudioSelectors, + validateStudioSelectors, + type StudioPreview, + type StudioValidationOutcome, +} from './studioService'; +import type { StudioIssue, StudioSuggestionState } from './studioModel'; +import { + emptyDraft, + selectorsFromDraft, + toggleSelectorParts, + withEnhance, + withFieldSelector, + withItemsSelector, + type StudioDraft, + type StudioFieldId, + type StudioSelectors, +} from './selectorDraft'; + +export const STUDIO_VALIDATE_DELAY_MS = 300; +export const STUDIO_PREVIEW_SETTLE_MS = 800; + +export type LivePreview = + | { readonly status: 'idle' } + | { readonly status: 'loading'; readonly held: StudioPreview | undefined } + | { readonly status: 'ready'; readonly sample: StudioPreview } + | { readonly status: 'failed'; readonly message: string }; + +interface StudioCore { + readonly draft: StudioDraft; + readonly yaml: string; + readonly live: LivePreview; + readonly validated: boolean; +} + +export type StudioState = + | (StudioCore & { readonly phase: 'editing' | 'validating' | 'saving' }) + | (StudioCore & { readonly phase: 'invalid'; readonly issues: readonly StudioIssue[] }) + | (StudioCore & { readonly phase: 'failed'; readonly message: string }); + +interface UseStudioOptions { + readonly url: string; + readonly token: string; + readonly validateDelayMs?: number; + readonly previewSettleMs?: number; +} + +export function useStudio({ + url, + token, + validateDelayMs = STUDIO_VALIDATE_DELAY_MS, + previewSettleMs = STUDIO_PREVIEW_SETTLE_MS, +}: UseStudioOptions) { + const [state, setState] = useState({ + phase: 'editing', + draft: emptyDraft(), + yaml: '', + validated: false, + live: { status: 'idle' }, + }); + const [suggestion, setSuggestion] = useState({ status: 'loading' }); + const [documentRevision, setDocumentRevision] = useState(0); + + const stateReference = useRef(state); + stateReference.current = state; + const requestIdReference = useRef(0); + const previewRequestReference = useRef(0); + const suggestRequestReference = useRef(0); + const suggestAbortReference = useRef(undefined); + const saveRequestReference = useRef(0); + const previewTimerReference = useRef | undefined>(undefined); + const previewAbortReference = useRef(undefined); + const previewSettleReference = useRef(previewSettleMs); + previewSettleReference.current = previewSettleMs; + const initialUrlReference = useRef(url); + + useEffect(() => { + const requestId = ++requestIdReference.current; + const controller = new AbortController(); + const revision = documentRevision; + const timer = setTimeout(() => { + void runValidate(requestId, controller.signal, revision); + }, validateDelayMs); + + return () => { + clearTimeout(timer); + controller.abort(); + }; + }, [documentRevision, token, url, validateDelayMs]); + + useEffect(() => { + const requestId = ++suggestRequestReference.current; + const controller = new AbortController(); + suggestAbortReference.current = controller; + setSuggestion({ status: 'loading' }); + void loadSuggestion(requestId, controller.signal); + + return () => { + controller.abort(); + }; + }, [url, token]); + + useEffect(() => { + return () => { + clearTimeout(previewTimerReference.current); + previewAbortReference.current?.abort(); + }; + }, []); + + async function loadSuggestion(requestId: number, signal: AbortSignal) { + try { + const evidence = await suggestStudioSelectors(token, url, signal); + if (signal.aborted || suggestRequestReference.current !== requestId) return; + setSuggestion(evidence); + } catch (error) { + if (isAbortError(error) || signal.aborted || suggestRequestReference.current !== requestId) return; + setSuggestion({ + status: 'failed', + message: error instanceof StudioRequestError ? error.message : COPY.unableToReachServer, + }); + } + } + + async function runValidate(requestId: number, signal: AbortSignal, revision: number) { + const sentDraft = stateReference.current.draft; + cancelPreview(); + setState((previous) => ({ ...coreOf(previous), phase: 'validating' })); + + try { + const outcome = await validateStudioSelectors(token, url, sentDraft, signal); + if (signal.aborted || requestIdReference.current !== requestId) return; + commitValidation(outcome, sentDraft, revision); + } catch (error) { + if (isAbortError(error) || signal.aborted || requestIdReference.current !== requestId) return; + setState((previous) => ({ + ...coreOf(previous), + phase: 'failed', + live: restoreHeld(previous.live), + message: error instanceof StudioRequestError ? error.message : COPY.unableToReachServer, + })); + } + } + + function commitValidation(outcome: StudioValidationOutcome, sentDraft: StudioDraft, revision: number) { + const draft = outcome.draft ? adoptServerDraft(sentDraft, outcome.draft) : sentDraft; + const yaml = outcome.yaml ?? stateReference.current.yaml; + + if (!outcome.valid) { + setState((previous) => ({ + ...coreOf(previous), + phase: 'invalid', + draft, + yaml, + issues: outcome.issues, + live: restoreHeld(previous.live), + })); + return; + } + + setState((previous) => ({ + ...coreOf(previous), + phase: 'editing', + draft, + yaml, + validated: true, + })); + if (revision === 0 && url === initialUrlReference.current) return; + schedulePreview(); + } + + function editSelectors(update: (draft: StudioDraft) => StudioDraft) { + cancelPreview(); + setState((previous) => ({ + ...coreOf(previous), + phase: 'editing', + draft: update(previous.draft), + validated: false, + live: refreshLive(previous.live), + })); + setDocumentRevision((revision) => revision + 1); + } + + function cancelPreview() { + clearTimeout(previewTimerReference.current); + previewAbortReference.current?.abort(); + previewAbortReference.current = undefined; + previewRequestReference.current += 1; + } + + function schedulePreview() { + cancelPreview(); + const requestId = previewRequestReference.current; + previewTimerReference.current = setTimeout(() => { + void runPreview(requestId); + }, previewSettleReference.current); + } + + async function runPreview(requestId: number) { + const current = stateReference.current; + if (!(current.phase === 'editing' && current.validated)) return; + if (previewRequestReference.current !== requestId) return; + + const controller = new AbortController(); + previewAbortReference.current = controller; + setState((previous) => { + if (!(previous.phase === 'editing' && previous.validated)) return previous; + if (previewRequestReference.current !== requestId) return previous; + return { + ...previous, + live: { + status: 'loading', + held: studioPreviewSample(previous.live), + }, + }; + }); + + try { + const sample = await previewStudioFeed(token, url, stateReference.current.draft, controller.signal); + if (controller.signal.aborted || previewRequestReference.current !== requestId) return; + if (sample.failure !== undefined) { + setState((previous) => withLive(previous, { status: 'failed', message: COPY.previewUnavailable })); + return; + } + setState((previous) => withLive(previous, { status: 'ready', sample })); + } catch (error) { + if (isAbortError(error) || controller.signal.aborted || previewRequestReference.current !== requestId) + return; + const message = error instanceof StudioRequestError ? error.message : COPY.unableToReachServer; + setState((previous) => withLive(previous, { status: 'failed', message })); + } + } + + return { + state, + suggestion, + isDirty: documentRevision > 0, + setItemsSelector: (itemsSelector: string) => { + editSelectors((draft) => withItemsSelector(draft, itemsSelector)); + }, + setEnhance: (isEnhanced: boolean) => { + cancelPreview(); + setState((previous) => { + const hasEnhanceFlipped = previous.draft.enhance !== isEnhanced; + return { + ...coreOf(previous), + phase: 'editing', + draft: withEnhance(previous.draft, isEnhanced), + validated: false, + live: hasEnhanceFlipped ? { status: 'idle' } : refreshLive(previous.live), + }; + }); + setDocumentRevision((revision) => revision + 1); + }, + setFieldSelector: (field: StudioFieldId, selector: string) => { + editSelectors((draft) => withFieldSelector(draft, field, selector)); + }, + toggleItemsSelector: (selector: string) => { + editSelectors((draft) => withItemsSelector(draft, toggleSelectorParts(draft.itemsSelector, selector))); + }, + toggleFieldSelector: (field: StudioFieldId, selector: string) => { + editSelectors((draft) => + withFieldSelector(draft, field, toggleSelectorParts(draft.fields[field].selector, selector)) + ); + }, + retrySuggestion: () => { + suggestAbortReference.current?.abort(); + const requestId = ++suggestRequestReference.current; + const controller = new AbortController(); + suggestAbortReference.current = controller; + setSuggestion({ status: 'loading' }); + void loadSuggestion(requestId, controller.signal); + }, + retryPreview: () => { + const current = stateReference.current; + if (!(current.phase === 'editing' && current.validated)) return; + cancelPreview(); + void runPreview(previewRequestReference.current); + }, + save: async (create: (selectors: StudioSelectors) => Promise) => { + const requestId = ++saveRequestReference.current; + const current = stateReference.current; + setState({ ...coreOf(current), phase: 'saving' }); + + try { + await create(selectorsFromDraft(current.draft)); + if (saveRequestReference.current !== requestId) return; + setState({ ...coreOf(current), phase: 'editing', validated: true }); + } catch (error) { + if (saveRequestReference.current !== requestId) return; + setState({ + ...coreOf(current), + phase: 'failed', + message: failureMessage(error), + }); + } + }, + }; +} + +export function studioIssues(state: StudioState): readonly StudioIssue[] { + return state.phase === 'invalid' ? state.issues : []; +} + +export function canSaveStudio(state: StudioState): boolean { + return state.phase === 'editing' && state.validated; +} + +export function studioPreviewSample(live: LivePreview): StudioPreview | undefined { + if (live.status === 'ready') return live.sample; + if (live.status === 'loading') return live.held; + return undefined; +} + +function coreOf(state: StudioState): StudioCore { + return { + draft: state.draft, + yaml: state.yaml, + live: state.live, + validated: state.validated, + }; +} + +function withLive(state: StudioState, live: LivePreview): StudioState { + return { ...state, live }; +} + +function refreshLive(live: LivePreview): LivePreview { + return { status: 'loading', held: studioPreviewSample(live) }; +} + +function restoreHeld(live: LivePreview): LivePreview { + const held = studioPreviewSample(live); + if (held === undefined) return { status: 'idle' }; + return { status: 'ready', sample: held }; +} + +function adoptServerDraft(sent: StudioDraft, incoming: StudioDraft): StudioDraft { + return { + ...sent, + itemsSelector: incoming.itemsSelector, + enhance: incoming.enhance, + }; +} + +function failureMessage(error: unknown): string { + if (isFeedCreationError(error)) return error.message; + if (error instanceof Error && error.message.trim()) return error.message; + return COPY.createFailedTitle; +} diff --git a/frontend/src/styles/main.css b/frontend/src/styles/main.css index 939359193..abfbc0c55 100644 --- a/frontend/src/styles/main.css +++ b/frontend/src/styles/main.css @@ -61,11 +61,6 @@ text-align: center; } -.workspace-shell--loading { - max-width: 32rem; - padding-top: var(--space-7); -} - .form-shell--minimal, .result-shell { width: 100%; @@ -164,8 +159,7 @@ display: none; } -.field-help, -.result-meta { +.field-help { margin: 0; color: var(--text-muted); text-align: center; @@ -177,10 +171,6 @@ color: var(--danger); } -.field-help--warning { - color: var(--text-body); -} - .catalog-hit-list { width: 100%; max-width: var(--layout-rail-reading); @@ -293,21 +283,6 @@ min-width: 0; } -.notice__code { - display: inline-block; - margin-top: var(--space-2); - padding: var(--space-1) var(--space-2); - background: rgb(var(--color-rgb-white) / 6%); - border: var(--border-width) solid var(--border-subtle); - border-radius: var(--radius-md); - font-family: var(--font-family-mono); - font-size: var(--font-size-00); - user-select: all; - cursor: pointer; - color: var(--text-strong); - width: fit-content; -} - .notice__title { color: var(--text-strong); font-weight: var(--font-weight-semibold); @@ -473,11 +448,6 @@ color: var(--text-strong); } -.utility-button--toggle { - letter-spacing: var(--eyebrow-letter-spacing); - text-transform: uppercase; -} - .app-footer { position: fixed; inset-inline: 0; @@ -572,6 +542,102 @@ } } +/* Reserved height so suggest loading → ready/empty/failed does not jump the field stack. */ +.studio-suggestion-status { + min-height: calc(var(--space-2) * 2 + 1.25rem); +} + +.studio-fields { + display: grid; + gap: var(--space-3); +} + +.studio-field { + display: grid; + gap: var(--space-2); +} + +.studio-candidates__chips { + display: flex; + flex-wrap: wrap; + gap: var(--space-2); + margin: 0; + padding: 0; + list-style: none; +} + +.studio-chip { + margin: 0; + min-width: 0; + max-width: 100%; + padding: var(--space-2) var(--space-3); + border: var(--border-width) solid transparent; + border-radius: var(--radius-pill); + background: transparent; + color: var(--text-body); + font-family: var(--font-family-mono); + font-size: var(--font-size-0); + line-height: var(--line-height-base); + overflow-wrap: anywhere; + cursor: pointer; +} + +.studio-chip:focus-visible { + outline: none; + box-shadow: var(--focus-ring); +} + +.studio-chip.is-pressed { + border-color: var(--border-strong); + background: var(--surface-elevated); + color: var(--text-strong); +} + +.studio-manual { + display: flex; + flex-wrap: wrap; + gap: var(--space-3); + align-items: center; +} + +.studio-enhance { + display: flex; + flex-direction: column; + align-items: stretch; + gap: var(--space-1); +} + +.studio-enhance__label { + display: flex; + align-items: center; + gap: var(--space-3); + margin: 0; + cursor: pointer; +} + +.studio-check { + flex: 0 0 auto; + width: var(--space-4); + height: var(--space-4); + margin: 0; +} + +.studio-actions { + display: flex; + flex-wrap: wrap; + gap: var(--space-3); + align-items: center; +} + +.studio-sample-image { + display: block; + width: 100%; + max-height: calc(var(--space-7) * 3); + margin: 0; + object-fit: cover; + border-radius: var(--radius-md); +} + @keyframes spin { to { transform: rotate(360deg); diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 89eeff6bf..986190e54 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -13,7 +13,7 @@ "isolatedModules": true, "noEmit": true, "strict": true, - "types": ["vitest/globals", "@testing-library/jest-dom", "msw"] + "types": ["@testing-library/jest-dom", "msw"] }, "include": ["src", "e2e", "playwright.config.ts", "vite.config.ts", "vitest*.ts", "vitest.config.js"] } diff --git a/frontend/vitest.config.js b/frontend/vitest.config.js index 3f4cab2ca..4ad87adf2 100644 --- a/frontend/vitest.config.js +++ b/frontend/vitest.config.js @@ -1,13 +1,58 @@ import { defineConfig, configDefaults } from 'vitest/config'; +const sharedExclude = [...configDefaults.exclude, 'tests/**', 'e2e/**']; + +const unitInclude = [ + 'src/__tests__/decideJourney.test.ts', + 'src/__tests__/catalog.test.ts', + 'src/__tests__/selectorDraft.test.ts', + 'src/__tests__/studioModel.test.ts', + 'src/__tests__/url.test.ts', + 'src/__tests__/feedCreationError.test.ts', + 'src/__tests__/feedWorkflowStorage.test.ts', + 'src/__tests__/studioService.test.ts', + 'src/__tests__/directoryHandoff.test.ts', + 'src/__tests__/previewHydration.test.ts', + 'src/__tests__/apiHttp.test.ts', + 'src/__tests__/persistentStorage.test.ts', + 'src/__tests__/bookmarkletHref.test.ts', +]; + +const integrationInclude = [ + 'src/__tests__/App.integration.test.tsx', + 'src/__tests__/ConfigStudio.test.tsx', + 'src/__tests__/ResultDisplay.test.tsx', + 'src/__tests__/useStudio.test.ts', + 'src/__tests__/useFeedCreation.test.ts', + 'src/__tests__/appRoute.test.ts', +]; + export default defineConfig({ test: { - environment: 'jsdom', - setupFiles: ['./src/__tests__/setup.ts'], - globals: true, + globals: false, testTimeout: 10_000, hookTimeout: 10_000, - exclude: [...configDefaults.exclude, 'tests/**', 'e2e/**'], + exclude: sharedExclude, + projects: [ + { + extends: true, + test: { + name: 'unit', + environment: 'node', + include: unitInclude, + setupFiles: ['./src/__tests__/setup.unit.ts'], + }, + }, + { + extends: true, + test: { + name: 'integration', + environment: 'jsdom', + include: integrationInclude, + setupFiles: ['./src/__tests__/setup.ts'], + }, + }, + ], }, esbuild: { jsx: 'automatic', diff --git a/frontend/vitest.contract.config.ts b/frontend/vitest.contract.config.ts deleted file mode 100644 index deea1c47d..000000000 --- a/frontend/vitest.contract.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { mergeConfig } from 'vitest/config'; -import baseConfig from './vitest.config.js'; - -export default mergeConfig(baseConfig, { - test: { - include: ['src/__tests__/**/*.contract.test.*'], - }, -}); diff --git a/frontend/vitest.unit.config.ts b/frontend/vitest.unit.config.ts deleted file mode 100644 index 2a54bdcac..000000000 --- a/frontend/vitest.unit.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { mergeConfig } from 'vitest/config'; -import baseConfig from './vitest.config.js'; - -export default mergeConfig(baseConfig, { - test: { - exclude: [...(baseConfig.test?.exclude ?? []), 'src/__tests__/**/*.contract.test.*'], - }, -}); diff --git a/public/openapi.yaml b/public/openapi.yaml index adafa5a9e..5f735394d 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -1,20 +1,408 @@ --- -openapi: 3.0.3 +components: + schemas: + FieldSelectorCandidate: + additionalProperties: false + description: One ranked field selector with a sample headline. Best match is + first. + properties: + sample: + description: Visible text of a matched node + type: string + selector: + type: string + required: + - selector + - sample + type: object + Html2rssExtractorAttribute: + const: attribute + description: Return the value of an HTML attribute on the selected element. + Requires sibling selector option `attribute` (attribute name). + examples: + - attribute + title: attribute + type: string + Html2rssExtractorHref: + const: href + description: Return the absolute URL from the selected element's `href` attribute + (relative hrefs are resolved against the page base URL). + examples: + - href + title: href + type: string + Html2rssExtractorHtml: + const: html + description: Return the outer HTML of the selected element. Sanitize during + post-processing (e.g. `sanitize_html`). + examples: + - html + title: html + type: string + Html2rssExtractorStatic: + const: static + description: Return a fixed value from sibling selector option `static` (no + DOM read). + examples: + - static + title: static + type: string + Html2rssExtractorText: + const: text + description: Return collapsed visible text of the selected element (default + extractor). + examples: + - text + title: text + type: string + Html2rssPostProcessorGsub: + additionalProperties: true + description: Replace matches of `pattern` in the extracted string with `replacement` + (Ruby String#gsub; pattern may be a regexp-like string). Patterns over 256 + characters as written, or with nested quantifiers, are rejected. + examples: + - name: gsub + pattern: boo + replacement: baz + properties: + name: + const: gsub + type: string + pattern: + maxLength: 256 + type: string + replacement: + type: + - string + - object + required: + - name + - pattern + - replacement + title: gsub + type: object + Html2rssPostProcessorMarkdownToHtml: + additionalProperties: true + description: Convert Markdown to HTML (Kramdown) and sanitize the result. Often + chained after `template`. + examples: + - name: markdown_to_html + properties: + name: + const: markdown_to_html + type: string + required: + - name + title: markdown_to_html + type: object + Html2rssPostProcessorParseTime: + additionalProperties: true + description: Parse a time string with Time.parse and return RFC822, using the + channel `time_zone`. + examples: + - name: parse_time + properties: + name: + const: parse_time + type: string + required: + - name + title: parse_time + type: object + Html2rssPostProcessorParseUri: + additionalProperties: true + description: Normalize a URL string; resolve relative URLs against the channel + URL. + examples: + - name: parse_uri + properties: + name: + const: parse_uri + type: string + required: + - name + title: parse_uri + type: object + Html2rssPostProcessorSanitizeHtml: + additionalProperties: true + description: 'Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute + URLs, safe link/img attributes, wrap lone images in anchors).' + examples: + - name: sanitize_html + properties: + name: + const: sanitize_html + type: string + required: + - name + title: sanitize_html + type: object + Html2rssPostProcessorSubstring: + additionalProperties: true + description: Return a slice of the extracted string using Integer `start` and + optional `end` (Ruby String#[] range semantics; end may be omitted). + examples: + - end: 6 + name: substring + start: 4 + properties: + end: + type: integer + name: + const: substring + type: string + start: + type: integer + required: + - name + - start + title: substring + type: object + Html2rssPostProcessorTemplate: + additionalProperties: true + description: Format a string with Kernel#format-style placeholders (`%{key}` + / `%s`). `%{self}` is the current selector value; other keys resolve + sibling selectors. + examples: + - name: template + string: "`%{self}` (`%{price}`)" + properties: + name: + const: template + type: string + string: + type: string + required: + - name + - string + title: template + type: object + ItemsSelectorCandidate: + additionalProperties: false + description: One ranked items selector with a sample headline. Best match is + first. + properties: + enhance: + type: boolean + sample: + description: Visible text of a matched node + type: string + selector: + type: string + required: + - selector + - enhance + - sample + type: object + PreviewSampleItem: + description: One preview meadow row returned by POST /feeds/preview. + properties: + image: + type: + - string + - 'null' + published_at: + type: + - string + - 'null' + title: + type: string + url: + type: + - string + - 'null' + required: + - title + type: object + SelectorCandidates: + additionalProperties: false + description: Ranked selector candidates. Empty buckets are a successful response. + properties: + items: + items: + "$ref": "#/components/schemas/ItemsSelectorCandidate" + type: array + link: + items: + "$ref": "#/components/schemas/FieldSelectorCandidate" + type: array + published: + items: + "$ref": "#/components/schemas/FieldSelectorCandidate" + type: array + title: + items: + "$ref": "#/components/schemas/FieldSelectorCandidate" + type: array + required: + - items + - title + - link + - published + type: object + SelectorsDocument: + additionalProperties: true + description: Selectors used to extract article attributes. + patternProperties: + "^(?!items$|enclosure$|guid$|categories$).+$": + description: Dynamic selector definition keyed by attribute name. + properties: + attribute: + minLength: 1 + type: string + extractor: + description: Extractor used to pull a value from the selected element. + oneOf: + - "$ref": "#/components/schemas/Html2rssExtractorAttribute" + - "$ref": "#/components/schemas/Html2rssExtractorHref" + - "$ref": "#/components/schemas/Html2rssExtractorHtml" + - "$ref": "#/components/schemas/Html2rssExtractorStatic" + - "$ref": "#/components/schemas/Html2rssExtractorText" + post_process: + description: Ordered transforms applied to the extracted value. + items: + oneOf: + - "$ref": "#/components/schemas/Html2rssPostProcessorGsub" + - "$ref": "#/components/schemas/Html2rssPostProcessorMarkdownToHtml" + - "$ref": "#/components/schemas/Html2rssPostProcessorParseTime" + - "$ref": "#/components/schemas/Html2rssPostProcessorParseUri" + - "$ref": "#/components/schemas/Html2rssPostProcessorSanitizeHtml" + - "$ref": "#/components/schemas/Html2rssPostProcessorSubstring" + - "$ref": "#/components/schemas/Html2rssPostProcessorTemplate" + type: array + static: + minLength: 1 + type: string + required: [] + type: object + properties: + categories: + description: List of selector keys whose values will be used as categories. + Each entry must reference a sibling selector key; runtime validation enforces + those references. + items: + description: Selector key defined elsewhere in this object. + type: string + minItems: 1 + type: array + enclosure: + description: Describes enclosure extraction settings. + properties: + attribute: + minLength: 1 + type: string + content_type: + minLength: 1 + pattern: "^[\\w-]+\\/[\\w-]+$" + type: string + extractor: + description: Extractor used to pull a value from the selected element. + oneOf: + - "$ref": "#/components/schemas/Html2rssExtractorAttribute" + - "$ref": "#/components/schemas/Html2rssExtractorHref" + - "$ref": "#/components/schemas/Html2rssExtractorHtml" + - "$ref": "#/components/schemas/Html2rssExtractorStatic" + - "$ref": "#/components/schemas/Html2rssExtractorText" + post_process: + description: Ordered transforms applied to the extracted value. + items: + oneOf: + - "$ref": "#/components/schemas/Html2rssPostProcessorGsub" + - "$ref": "#/components/schemas/Html2rssPostProcessorMarkdownToHtml" + - "$ref": "#/components/schemas/Html2rssPostProcessorParseTime" + - "$ref": "#/components/schemas/Html2rssPostProcessorParseUri" + - "$ref": "#/components/schemas/Html2rssPostProcessorSanitizeHtml" + - "$ref": "#/components/schemas/Html2rssPostProcessorSubstring" + - "$ref": "#/components/schemas/Html2rssPostProcessorTemplate" + type: array + static: + minLength: 1 + type: string + required: [] + type: object + guid: + description: List of selector keys used to build the GUID. Each entry must + reference a sibling selector key; runtime validation enforces those references. + items: + description: Selector key defined elsewhere in this object. + type: string + minItems: 1 + type: array + items: + description: Defines the items selector and list-card enhance settings. + properties: + enhance: + description: 'List-card enrichment: run Html::ArticleExtractor on each + matched item node to fill missing fields from the card HTML.' + not: + type: 'null' + type: boolean + order: + enum: + - reverse + not: + type: 'null' + pagination: + description: Pagination configuration or maximum page count integer. + oneOf: + - exclusiveMinimum: 0 + type: integer + - additionalProperties: true + properties: + cursor_path: + type: string + increment: + exclusiveMinimum: 0 + type: integer + max_pages: + exclusiveMinimum: 0 + type: integer + next_url_path: + type: string + param: + type: string + selector: + type: string + start_offset: + type: integer + start_page: + type: integer + step: + exclusiveMinimum: 0 + type: integer + strategy: + enum: + - rel_next + - custom_selector + - url_template + - offset + - json_cursor + type: string + type: object + selector: + default: a[href] + minLength: 1 + type: string + required: [] + type: object + type: object + securitySchemes: + BearerAuth: + bearerFormat: JWT + description: Bearer token authentication for API access. + scheme: bearer + type: http info: - title: html2rss-web API - version: 1.13.0 - description: RESTful API for converting websites to RSS feeds. contact: name: html2rss-web Support url: https://github.com/html2rss/html2rss-web + description: RESTful API for converting websites to RSS feeds. license: name: MIT url: https://opensource.org/licenses/MIT -servers: -- url: https://api.html2rss.dev/api/v1 - description: Production server -- url: http://127.0.0.1:4000/api/v1 - description: Development server + title: html2rss-web API + version: 1.13.0 +openapi: 3.1.0 paths: "/": get: @@ -64,8 +452,16 @@ paths: - enabled - access_token_required type: object + studio: + properties: + enabled: + type: boolean + required: + - enabled + type: object required: - feed_creation + - studio - catalog type: object required: @@ -78,7 +474,7 @@ paths: - success - data type: object - description: returns catalog pointer metadata + description: API metadata security: - {} summary: API metadata @@ -132,11 +528,13 @@ paths: last_result: properties: at: - nullable: true - type: string + type: + - string + - 'null' code: - nullable: true - type: string + type: + - string + - 'null' state: type: string required: @@ -149,85 +547,99 @@ paths: defaults: properties: blog: - nullable: true - type: string + type: + - string + - 'null' id: - nullable: true - type: string + type: + - string + - 'null' region: - nullable: true - type: string + type: + - string + - 'null' repository: - nullable: true - type: string + type: + - string + - 'null' section: - nullable: true - type: string + type: + - string + - 'null' user_id: - nullable: true - type: string + type: + - string + - 'null' username: - nullable: true - type: string + type: + - string + - 'null' type: object schema: properties: blog: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' id: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' region: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' repository: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' section: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' user_id: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' username: - nullable: true properties: type: type: string required: - type - type: object + type: + - object + - 'null' type: object required: - schema @@ -272,7 +684,7 @@ paths: - data - meta type: object - description: returns the merged catalog with CORS headers + description: Config catalog '404': content: application/json: @@ -283,7 +695,7 @@ paths: required: - error type: object - description: returns 404 when the catalog is disabled + description: Config catalog security: - {} summary: Config catalog @@ -300,7 +712,7 @@ paths: type: object responses: '204': - description: responds to preflight requests with CORS headers + description: Config catalog preflight security: - {} summary: Config catalog preflight @@ -323,6 +735,8 @@ paths: properties: name: type: string + selectors: + "$ref": "#/components/schemas/SelectorsDocument" url: type: string required: @@ -381,7 +795,7 @@ paths: - data - meta type: object - description: normalizes hostname-only input to https before feed creation + description: Create a feed '400': content: application/json: @@ -415,7 +829,7 @@ paths: - success - error type: object - description: returns 400 when request body exceeds maximum allowed bytes + description: Create a feed '401': content: application/json: @@ -449,7 +863,7 @@ paths: - success - error type: object - description: returns 401 with UNAUTHORIZED error payload + description: Create a feed '403': content: application/json: @@ -483,8 +897,7 @@ paths: - success - error type: object - description: returns forbidden for authenticated requests when auto source - is disabled + description: Create a feed '429': content: application/json: @@ -518,10 +931,9 @@ paths: - success - error type: object - description: returns 429 when rate limit is exceeded + description: Create a feed headers: Retry-After: - description: The number of seconds to wait before retrying the request. schema: type: integer security: @@ -529,6 +941,342 @@ paths: summary: Create a feed tags: - Feeds + "/feeds/preview": + post: + description: Preview a selectors document feed + operationId: previewFeed + parameters: + - in: header + name: Authorization + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + selectors: + "$ref": "#/components/schemas/SelectorsDocument" + url: + type: string + required: + - url + - selectors + type: object + responses: + '200': + content: + application/json: + schema: + properties: + data: + properties: + failure_kind: + type: + - string + - 'null' + item_count: + type: integer + quality_report: + properties: + metrics: + properties: + short_title_count: + type: integer + required: + - short_title_count + type: object + warnings: + items: + type: string + type: array + required: + - warnings + - metrics + type: + - object + - 'null' + sample_items: + items: + "$ref": "#/components/schemas/PreviewSampleItem" + type: array + validation_issues: + items: {} + type: array + required: + - item_count + - sample_items + - quality_report + - failure_kind + - validation_issues + type: object + success: + type: boolean + required: + - success + - data + type: object + description: Preview a selectors document feed + security: + - BearerAuth: [] + summary: Preview a selectors document feed + tags: + - Studio + "/feeds/suggest_selectors": + post: + description: Suggest selectors for a page + operationId: suggestSelectors + parameters: + - in: header + name: Authorization + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + items_selector: + type: string + url: + type: string + required: + - url + - items_selector + type: object + responses: + '200': + content: + application/json: + schema: + properties: + data: + properties: + admission_drops: + properties: + chrome: + type: integer + type: object + candidates: + "$ref": "#/components/schemas/SelectorCandidates" + segment_strategy: + type: string + required: + - candidates + - segment_strategy + - admission_drops + type: object + success: + type: boolean + required: + - success + - data + type: object + description: Suggest selectors for a page + '422': + content: + application/json: + schema: + properties: + error: + properties: + code: + type: string + kind: + type: string + message: + type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean + required: + - code + - message + - kind + - retryable + - next_action + - retry_action + type: object + success: + type: boolean + required: + - success + - error + type: object + description: Suggest selectors for a page + security: + - BearerAuth: [] + summary: Suggest selectors for a page + tags: + - Studio + "/feeds/validate": + post: + description: Validate a selectors document + operationId: validateFeedConfig + parameters: + - in: header + name: Authorization + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + selectors: + "$ref": "#/components/schemas/SelectorsDocument" + url: + type: string + yaml: + type: string + type: object + responses: + '200': + content: + application/json: + schema: + properties: + data: + properties: + report: + properties: + issues: + items: + properties: + actual: + description: JSON-ish value, or null when the issue + carries none. + code: + enum: + - constraint + - invalid_value + - missing_key + - parse + - type_mismatch + - unknown_value + type: string + expected: + description: JSON-ish value, or null when the issue + carries none. + message: + type: string + path: + items: + type: string + type: array + required: + - path + - code + - message + - expected + - actual + type: object + type: array + success: + type: boolean + required: + - success + - issues + type: object + selectors: + oneOf: + - type: 'null' + - "$ref": "#/components/schemas/SelectorsDocument" + yaml: + type: + - string + - 'null' + required: + - report + - selectors + - yaml + type: object + success: + type: boolean + required: + - success + - data + type: object + description: Validate a selectors document + '400': + content: + application/json: + schema: + properties: + error: + properties: + code: + type: string + kind: + type: string + message: + type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean + required: + - code + - message + - kind + - retryable + - next_action + - retry_action + type: object + success: + type: boolean + required: + - success + - error + type: object + description: Validate a selectors document + '401': + content: + application/json: + schema: + properties: + error: + properties: + code: + type: string + kind: + type: string + message: + type: string + next_action: + type: string + retry_action: + type: string + retryable: + type: boolean + required: + - code + - message + - kind + - retryable + - next_action + - retry_action + type: object + success: + type: boolean + required: + - success + - error + type: object + description: Validate a selectors document + security: + - BearerAuth: [] + summary: Validate a selectors document + tags: + - Studio "/feeds/{token}": get: description: Render feed by token @@ -548,70 +1296,100 @@ paths: application/xml: schema: type: string - description: renders feed for a valid token + description: Render feed by token + headers: + Cache-Control: + schema: + type: string + Link: + schema: + type: string + Vary: + schema: + type: string '401': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns unauthorized for invalid tokens + description: Render feed by token + headers: + Cache-Control: + schema: + type: string '403': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns forbidden when auto source is disabled + description: Render feed by token + headers: + Cache-Control: + schema: + type: string '422': content: text/plain: - example: We could not extract feed items from this page yet. Try a more - specific listing URL or explicit selectors. schema: type: string - description: returns 422 for empty extraction feeds in xml representation + description: Render feed by token + headers: + Cache-Control: + schema: + type: string + Vary: + schema: + type: string '429': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns 429 when rate limit is exceeded + description: Render feed by token headers: + Cache-Control: + schema: + type: string Retry-After: - description: The number of seconds to wait before retrying the request. schema: type: integer '500': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns non-cacheable feed errors when service generation fails + description: Render feed by token + headers: + Cache-Control: + schema: + type: string + Vary: + schema: + type: string '503': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns 503 when the scraper queue times out + description: Render feed by token headers: + Cache-Control: + schema: + type: string Retry-After: - description: The number of seconds to wait before retrying the request. schema: type: integer '504': content: text/plain: - example: 'Failed to generate feed: Invalid token' schema: type: string - description: returns 504 when the gateway times out + description: Render feed by token headers: + Cache-Control: + schema: + type: string Retry-After: - description: The number of seconds to wait before retrying the request. schema: type: integer security: @@ -667,7 +1445,7 @@ paths: - success - data type: object - description: returns health status when token is valid + description: Authenticated health check '401': content: application/json: @@ -701,7 +1479,7 @@ paths: - success - error type: object - description: returns 401 with UNAUTHORIZED error payload + description: Authenticated health check '500': content: application/json: @@ -735,7 +1513,7 @@ paths: - success - error type: object - description: returns error when configuration fails + description: Authenticated health check security: - BearerAuth: [] summary: Authenticated health check @@ -772,7 +1550,7 @@ paths: - success - data type: object - description: returns liveness status without authentication + description: Liveness probe security: - {} summary: Liveness probe @@ -820,7 +1598,7 @@ paths: - success - data type: object - description: returns readiness status without authentication + description: Readiness probe security: - {} summary: Readiness probe @@ -870,27 +1648,27 @@ paths: - data - meta type: object - description: returns available strategies + description: List extraction strategies security: - {} summary: List extraction strategies tags: - Strategies -components: - securitySchemes: - BearerAuth: - description: Bearer token authentication for API access. - type: http - scheme: bearer - bearerFormat: JWT +servers: +- description: Production server + url: https://api.html2rss.dev/api/v1 +- description: Development server + url: http://127.0.0.1:4000/api/v1 tags: -- name: Root - description: API metadata and service-level information. -- name: Catalog - description: Public feed-directory catalog metadata. -- name: Health - description: Health and readiness endpoints. -- name: Strategies - description: Feed extraction strategy discovery. -- name: Feeds - description: Feed creation and feed rendering operations. +- description: API metadata and service-level information. + name: Root +- description: Public feed-directory catalog metadata. + name: Catalog +- description: Health and readiness endpoints. + name: Health +- description: Feed extraction strategy discovery. + name: Strategies +- description: Feed creation and feed rendering operations. + name: Feeds +- description: Validate, preview, and suggest refined feed configs. + name: Studio diff --git a/public/shared-ui.css b/public/shared-ui.css index b1c819151..f44be11c4 100644 --- a/public/shared-ui.css +++ b/public/shared-ui.css @@ -181,9 +181,7 @@ textarea { } .layout-shell-padding { - padding: - clamp(0.85rem, 3vh, 2rem) - clamp(var(--space-3), 3vw, var(--space-4)) + padding: clamp(0.85rem, 3vh, 2rem) clamp(var(--space-3), 3vw, var(--space-4)) var(--layout-shell-padding-bottom, var(--space-5)); } @@ -302,6 +300,11 @@ textarea { border-top: var(--border-width) solid var(--border-subtle); } +.ui-item-list--grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + column-gap: var(--space-4); +} + .ui-item { min-width: 0; padding: var(--space-3) 0; @@ -381,7 +384,10 @@ textarea { min-height: var(--control-input-lg-min-height, 4.5rem); padding-inline: var(--control-input-lg-padding-inline, 1.25rem 5rem); font-size: var(--control-input-lg-font-size, clamp(1.05rem, 2.3vw, 1.5rem)); - letter-spacing: var(--control-input-lg-letter-spacing, var(--letter-spacing-display)); + letter-spacing: var( + --control-input-lg-letter-spacing, + var(--letter-spacing-display) + ); } .input:disabled { @@ -518,4 +524,8 @@ textarea { grid-template-columns: 1fr; width: 100%; } + + .ui-item-list--grid { + grid-template-columns: 1fr; + } } diff --git a/spec/html2rss/web/api/studio_auth_chain_spec.rb b/spec/html2rss/web/api/studio_auth_chain_spec.rb new file mode 100644 index 000000000..62856082e --- /dev/null +++ b/spec/html2rss/web/api/studio_auth_chain_spec.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'base64' +require 'climate_control' +require 'json' +require 'zlib' + +# Abuse-path coverage for the live auth chain behind the config studio. +# +# Every example drives a real Rack request through the full middleware stack, so +# a gate that is bypassed at the middleware or routing layer still fails here. +## +# Rack input that records how many bytes anything upstream actually consumed. +class CountingInput < StringIO + # @return [Integer] + attr_reader :bytes_read + + # @param string [String] + def initialize(string) + super + @bytes_read = 0 + end + + # @return [String, nil] + def read(...) + super.tap { @bytes_read += it.to_s.bytesize } + end +end + +RSpec.describe 'api/v1 studio auth chain', openapi: false, type: :request do + include Rack::Test::Methods + + # Restricted account from config/feeds.yml; admin is allowlisted for "*". + let(:member_token) { 'CHANGE_ME_DEMO_TOKEN' } + let(:member_url) { 'https://news.ycombinator.com' } + let(:outside_url) { 'https://evil.example/articles' } + let(:selectors) { { items: { selector: 'article' } } } + + def app = Html2rss::Web::App.freeze.app + + def json_post(path, payload, token: nil) + header 'Authorization', "Bearer #{token}" if token + header 'Content-Type', 'application/json' + post path, payload.to_json + end + + def error_message(response) = JSON.parse(response.body).dig('error', 'message') + + # Signs a token directly so the example can bind a URL the create endpoint + # would have refused, which is the state left behind by an allowlist change. + def mint(username, url, selectors: nil, expires_in: 3600) + Html2rss::Web::Auth.generate_feed_token(username, url, selectors:, expires_in:) + end + + def unpack(encoded) = JSON.parse(Zlib::Inflate.inflate(Base64.urlsafe_decode64(encoded)), symbolize_names: true) + def pack(document) = Base64.urlsafe_encode64(Zlib::Deflate.deflate(JSON.generate(document))) + + describe 'broken access control' do + it 'refuses to serve a feed token bound to a URL outside the account allowlist' do + get "/api/v1/feeds/#{mint('demo', outside_url)}" + + expect(last_response.status).to eq(403) + end + + it 'refuses to preview a URL outside the account allowlist' do + json_post '/api/v1/feeds/preview', { url: outside_url, selectors: }, token: member_token + + expect(last_response.status).to eq(403) + expect(error_message(last_response)).to eq('URL not allowed for this account') + end + + it 'refuses to suggest selectors for a URL outside the account allowlist' do + json_post '/api/v1/feeds/suggest_selectors', { url: outside_url }, token: member_token + + expect(last_response.status).to eq(403) + expect(error_message(last_response)).to eq('URL not allowed for this account') + end + + it 'rejects a valid token replayed under another username' do + document = unpack(mint('demo', member_url)) + document[:p][:u] = 'admin' + + get "/api/v1/feeds/#{pack(document)}" + + expect(last_response.status).to eq(401) + end + + it 'does not reach a static feed by putting its name in the token route' do + get '/api/v1/feeds/example' + + expect(last_response.status).to eq(401) + end + end + + describe 'cryptographic integrity' do + before { allow(Html2rss).to receive(:feed_result).and_call_original } + + it 'rejects a flipped byte in the signed selectors payload and never fetches', :aggregate_failures do + signed = mint('demo', member_url, selectors: Html2rss::Web::SelectorsDocument.from_client({ selectors: })) + document = unpack(signed) + document[:p][:c][:selectors][:items][:selector] = 'aRticle' + + get "/api/v1/feeds/#{pack(document)}" + + expect(last_response.status).to eq(401) + expect(Html2rss).not_to have_received(:feed_result) + end + + it 'rejects an expired token' do + get "/api/v1/feeds/#{mint('demo', member_url, expires_in: -60)}" + + expect(last_response.status).to eq(401) + end + + it 'does not run gem schema validation before the signature check' do + allow(Html2rss::Config).to receive(:validate).and_call_original + unsigned = pack( + p: { u: 'demo', l: member_url, e: Time.now.to_i + 3600, c: { selectors: } }, + s: 'forged-signature' + ) + + get "/api/v1/feeds/#{unsigned}" + + expect(Html2rss::Config).not_to have_received(:validate) + end + + it 'answers 401 rather than 500 for an undecodable token' do + get '/api/v1/feeds/not-a-token-at-all' + + expect(last_response.status).to eq(401) + end + end + + describe 'denial of service' do + it 'discards a compression bomb instead of inflating it', :aggregate_failures do + bomb = Base64.urlsafe_encode64( + Zlib::Deflate.deflate({ p: { u: 'demo', l: member_url, e: 1, pad: 'A' * 20_000_000 }, s: 'x' }.to_json, 9) + ) + + expect(Html2rss::Web::FeedToken::Codec.decode(bomb)).to be_nil + expect(bomb.bytesize).to be > Html2rss::Web::FeedToken::MAX_ENCODED_BYTES + end + + it 'rejects an oversized encoded token' do + get "/api/v1/feeds/#{'A' * (Html2rss::Web::FeedToken::MAX_ENCODED_BYTES + 1)}" + + expect(last_response.status).to eq(401) + end + + it 'rejects an oversized form body without reading it', :aggregate_failures do + body = "x=#{'A' * (200 * 1024)}" + input = CountingInput.new(body) + status, = app.call( + Rack::MockRequest.env_for( + '/api/v1/feeds/validate', + method: 'POST', input:, 'CONTENT_TYPE' => 'application/x-www-form-urlencoded', + 'CONTENT_LENGTH' => body.bytesize.to_s, 'HTTP_AUTHORIZATION' => "Bearer #{member_token}" + ) + ) + + expect(body.bytesize).to be > Html2rss::Web::Api::V1::JsonBody::MAX_BODY_BYTES + expect(status).to eq(400) + expect(input.bytes_read).to eq(0) + end + + it 'rejects an oversized items_selector hint' do + json_post '/api/v1/feeds/suggest_selectors', + { url: member_url, items_selector: 'a' * 5_000 }, + token: member_token + + expect(last_response.status).to eq(400) + expect(error_message(last_response)).to eq('items_selector is too long') + end + end + + describe 'server-side request forgery' do + %w[channel headers strategy request].each do |denied| + it "rejects a selectors document payload carrying #{denied}" do + json_post '/api/v1/feeds/validate', + { denied => { 'url' => 'http://169.254.169.254/' }, 'selectors' => selectors }, + token: member_token + + expect(last_response.status).to eq(400) + expect(error_message(last_response)).to eq("#{denied} is not allowed") + end + end + + it 'keeps UrlValidator as the only decider of the fetched URL' do + token = mint('demo', member_url, selectors: Html2rss::Web::SelectorsDocument.from_client({ selectors: })) + resolved = Html2rss::Web::Feeds::SourceResolver.call( + Html2rss::Web::Feeds::Contracts::Request.new(target_kind: :token, feed_name: nil, token:, params: {}) + ) + + expect(resolved.generator_input[:channel]).to eq(url: member_url) + end + + it 'reports a root key gate rejection as 400 even when the key name mimics an issue line' do + json_post '/api/v1/feeds/validate', + { yaml: "foo [bar: 1\nselectors:\n items:\n selector: article\n" }, + token: member_token + + expect(last_response.status).to eq(400) + expect(error_message(last_response)).to eq('foo [bar is not allowed') + end + + it 'still reports a schema failure as a structured issue', :aggregate_failures do + json_post '/api/v1/feeds/validate', { selectors: { items: { selector: 42 } } }, token: member_token + + json = expect_success_response(last_response) + expect(json.dig('data', 'report', 'success')).to be(false) + expect(json.dig('data', 'report', 'issues')).not_to be_empty + end + + it 'reports unparseable yaml as a parse issue rather than a 500' do + json_post '/api/v1/feeds/validate', { yaml: "selectors: [\n" }, token: member_token + + expect(JSON.parse(last_response.body).dig('data', 'report', 'issues', 0, 'code')).to eq('parse') + end + end + + describe 'authentication and misconfiguration' do + %w[feeds feeds/validate feeds/preview feeds/suggest_selectors].each do |path| + it "rejects an unauthenticated POST to /api/v1/#{path}" do + json_post "/api/v1/#{path}", { url: member_url, selectors: } + + expect(last_response.status).to eq(401) + end + end + + it 'authenticates before disclosing that the studio is disabled' do + ClimateControl.modify(STUDIO_ENABLED: 'false') do + json_post '/api/v1/feeds/validate', { selectors: } + end + + expect(last_response.status).to eq(401) + end + + it 'authenticates before disclosing that auto source is disabled' do + ClimateControl.modify(AUTO_SOURCE_ENABLED: 'false') do + json_post '/api/v1/feeds/preview', { url: member_url, selectors: } + end + + expect(last_response.status).to eq(401) + end + + it 'fails the studio and auto-source flags closed when RACK_ENV is unset', :aggregate_failures do + ClimateControl.modify(RACK_ENV: nil) do + expect(Html2rss::Web::Flags.studio_enabled?).to be(false) + expect(Html2rss::Web::Flags.auto_source_enabled?).to be(false) + end + end + + it 'keeps studio route names readable in the audit trail', :aggregate_failures do + Html2rss::Web::Routes::ApiV1::FeedRoutes::STUDIO_POSTS.each_key do |name| + expect(Html2rss::Web::LogSanitizer.sanitize_path("/api/v1/feeds/#{name}")).to eq("/api/v1/feeds/#{name}") + end + expect(Html2rss::Web::LogSanitizer.sanitize_path("/api/v1/feeds/#{mint('demo', member_url)}")) + .to eq('/api/v1/feeds/[REDACTED]') + end + end +end diff --git a/spec/html2rss/web/api/v1/json_body_spec.rb b/spec/html2rss/web/api/v1/json_body_spec.rb new file mode 100644 index 000000000..68bfe9e58 --- /dev/null +++ b/spec/html2rss/web/api/v1/json_body_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'stringio' + +RSpec.describe Html2rss::Web::Api::V1::JsonBody do + def request_for(body, content_length: body.bytesize) + env = Rack::MockRequest.env_for( + '/api/v1/feeds', + method: 'POST', + input: body, + 'CONTENT_TYPE' => 'application/json', + 'CONTENT_LENGTH' => content_length.to_s + ) + Rack::Request.new(env) + end + + it 'parses a JSON object and rewinds the body', :aggregate_failures do + request = request_for('{"url":"https://example.com"}') + + expect(described_class.object(request)).to eq('url' => 'https://example.com') + expect(request.body.read).to eq('{"url":"https://example.com"}') + end + + it 'treats an empty body as an empty object' do + expect(described_class.object(request_for(''))).to eq({}) + end + + it 'rejects a JSON array' do + expect { described_class.object(request_for('[1]')) } + .to raise_error(Html2rss::Web::BadRequestError, 'Invalid JSON payload') + end + + it 'rejects a declared length over the cap without reading', :aggregate_failures do + body = StringIO.new('{"url":"https://example.com"}') + request = request_for(body, content_length: described_class::MAX_BODY_BYTES + 1) + + expect { described_class.object(request) } + .to raise_error(Html2rss::Web::BadRequestError, 'Payload too large') + expect(body.pos).to eq(0) + end + + it 'rejects a body longer than the cap when length is understated' do + raw = 'x' * (described_class::MAX_BODY_BYTES + 1) + + expect { described_class.object(request_for(raw, content_length: 1)) } + .to raise_error(Html2rss::Web::BadRequestError, 'Payload too large') + end +end diff --git a/spec/html2rss/web/api/v1/preview_samples_spec.rb b/spec/html2rss/web/api/v1/preview_samples_spec.rb new file mode 100644 index 000000000..9eb0db4a2 --- /dev/null +++ b/spec/html2rss/web/api/v1/preview_samples_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Html2rss::Web::Api::V1::PreviewSamples do + def preview_result(rss:, sample_items: []) + Struct.new(:rss, :sample_items).new(rss, sample_items) + end + + def rss_item(index, extra = '') + %(Item #{index}https://example.com/#{index}#{extra}) + end + + def rss_document(items) + %(t#{items}) + end + + def image_rss + items = [ + rss_item(1, '<img src="https://cdn.example/1.jpg">'), + rss_item(2, '<img src="javascript:alert(1)">'), + rss_item(3, ''), + rss_item(4, '') + ] + rss_document(items.join) + end + + it 'maps rss rows and keeps only safe http images', :aggregate_failures do + rows = described_class.from_result(preview_result(rss: image_rss)) + + expect(rows.map { it['image'] }).to eq( + ['https://cdn.example/1.jpg', nil, nil, 'http://cdn.example/4.jpg'] + ) + expect(rows[1]).not_to have_key('image') + expect(rows[0].keys - described_class::SAMPLE_KEYS).to be_empty + end + + it 'caps mapped rows at the sample limit', :aggregate_failures do + items = (1..12).map { rss_item(it) }.join + rows = described_class.from_result(preview_result(rss: rss_document(items))) + + expect(rows.length).to eq(described_class::SAMPLE_LIMIT) + expect(rows.last['title']).to eq('Item 10') + end + + it 'keeps gem samples when the rss document cannot be parsed', :aggregate_failures do + gem_samples = [{ 'title' => 'A' }] + + expect(described_class.from_result(preview_result(rss: ' true) + end + it 'returns catalog pointer metadata', :aggregate_failures do get '/api/v1' @@ -217,6 +226,7 @@ def relative_feed_link_header(token) describe 'GET /api/v1/configs', openapi: { summary: 'Config catalog', + description: 'Config catalog', operation_id: 'getConfigCatalog', tags: ['Catalog'], security: [{}] @@ -292,6 +302,7 @@ def relative_feed_link_header(token) describe 'OPTIONS /api/v1/configs', openapi: { summary: 'Config catalog preflight', + description: 'Config catalog preflight', operation_id: 'optionsConfigCatalog', tags: ['Catalog'], security: [{}] @@ -332,6 +343,7 @@ def relative_feed_link_header(token) describe 'GET /api/v1/health', openapi: { summary: 'Authenticated health check', + description: 'Authenticated health check', operation_id: 'getHealthStatus', tags: ['Health'], security: [{ 'BearerAuth' => [] }] @@ -408,6 +420,7 @@ def relative_feed_link_header(token) describe 'GET /api/v1/health/ready', openapi: { summary: 'Readiness probe', + description: 'Readiness probe', operation_id: 'getReadinessProbe', tags: ['Health'], security: [{}] @@ -424,6 +437,7 @@ def relative_feed_link_header(token) describe 'GET /api/v1/health/live', openapi: { summary: 'Liveness probe', + description: 'Liveness probe', operation_id: 'getLivenessProbe', tags: ['Health'], security: [{}] @@ -440,6 +454,7 @@ def relative_feed_link_header(token) describe 'GET /api/v1/strategies', openapi: { summary: 'List extraction strategies', + description: 'List extraction strategies', operation_id: 'listStrategies', tags: ['Strategies'], security: [{}] @@ -458,10 +473,10 @@ def relative_feed_link_header(token) describe 'GET /api/v1/feeds/:token', openapi: { summary: 'Render feed by token', + description: 'Render feed by token', operation_id: 'renderFeedByToken', tags: ['Feeds'], - security: [{}], - example_mode: :multiple + security: [{}] } do before do stub_const('Html2rss::FeedChannel', Class.new { attr_reader :ttl }) @@ -694,6 +709,7 @@ def relative_feed_link_header(token) describe 'POST /api/v1/feeds', openapi: { summary: 'Create a feed', + description: 'Create a feed', operation_id: 'createFeed', tags: ['Feeds'], security: [{ 'BearerAuth' => [] }] @@ -728,6 +744,29 @@ def relative_feed_link_header(token) retry_action: 'none' } + it 'signs optional selectors into the feed token', :aggregate_failures do + selectors = { + items: { selector: 'article', enhance: true }, + title: { selector: 'h2', extractor: 'text' }, + url: { selector: 'a', extractor: 'href' }, + published_at: { selector: 'time', extractor: 'text' } + } + post_feed_request(url: feed_url, selectors:) + + token = expect_success_response(last_response).dig('data', 'feed', 'feed_token') + fragment = Html2rss::Web::Auth.validate_and_decode_feed_token(token).selectors.to_config_fragment + expect(fragment.dig(:selectors, :items, :selector)).to eq('article') + expect(fragment.dig(:selectors, :title, :extractor)).to eq('text') + end + + it 'signs pagination selectors into the feed token', :aggregate_failures do + post_feed_request(url: feed_url, selectors: SelectorsContractFixtures::PAGINATION_INTEGER) + + token = expect_success_response(last_response).dig('data', 'feed', 'feed_token') + fragment = Html2rss::Web::Auth.validate_and_decode_feed_token(token).selectors.to_config_fragment + expect(fragment.dig(:selectors, :items, :pagination)).to eq(2) + end + it 'creates a feed when request is valid', :aggregate_failures do header 'Authorization', "Bearer #{admin_token}" header 'Content-Type', 'application/json' @@ -790,6 +829,24 @@ def relative_feed_link_header(token) expect(json.dig('error', 'message')).to eq(Html2rss::Web::AutoSourceDisabledError::DEFAULT_MESSAGE) end + it 'rejects client-authored selectors while the studio is disabled', :aggregate_failures, openapi: false do + ClimateControl.modify(STUDIO_ENABLED: 'false') do + post_feed_request(url: feed_url, selectors: { items: { selector: 'article' } }) + end + + expect(last_response.status).to eq(403) + expect(JSON.parse(last_response.body).dig('error', 'message')).to eq('Studio is disabled') + end + + it 'still creates a url-only feed while the studio is disabled', :aggregate_failures, openapi: false do + ClimateControl.modify(STUDIO_ENABLED: 'false', AUTO_SOURCE_ENABLED: 'true') do + post_feed_request(url: feed_url) + end + + expect(last_response.status).to eq(201) + expect(expect_success_response(last_response).dig('data', 'feed', 'url')).to eq(feed_url) + end + it 'returns 429 when rate limit is exceeded', :aggregate_failures do allow(Html2rss::Web::Flags).to receive_messages( rate_limit_enabled?: true, @@ -832,4 +889,465 @@ def relative_feed_link_header(token) expect(json.dig('error', 'message')).to eq('Payload too large') end end + + describe 'POST /api/v1/feeds/validate', openapi: { + summary: 'Validate a selectors document', + description: 'Validate a selectors document', + operation_id: 'validateFeedConfig', + tags: ['Studio'], + security: [{ 'BearerAuth' => [] }] + } do + def post_validate(payload) + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/validate', payload.to_json + end + + let(:perform_request) do + lambda { + header 'Content-Type', 'application/json' + post '/api/v1/feeds/validate', { + selectors: { items: { selector: 'article' } }, + url: 'https://example.com/articles' + }.to_json + } + end + + it_behaves_like 'api error contract', { + status: 401, + code: Html2rss::Web::UnauthorizedError::CODE, + kind: 'auth', + retryable: false, + next_action: 'enter_token', + retry_action: 'none' + } + + it 'returns a structured report for a selectors document', :aggregate_failures do + selectors = { + items: { selector: 'article', enhance: true }, + title: { selector: 'h2', extractor: 'text' }, + url: { selector: 'a', extractor: 'href' }, + published_at: { selector: 'time', extractor: 'text' } + } + post_validate({ selectors:, url: 'https://example.com/articles' }) + + json = expect_success_response(last_response) + expect(json.dig('data', 'report', 'success')).to be(true) + expect(json.dig('data', 'selectors', 'items', 'selector')).to eq('article') + expect(json.dig('data', 'selectors', 'title', 'extractor')).to eq('text') + expect(json.dig('data', 'selectors', 'url', 'extractor')).to eq('href') + end + + it 'exports the page url beside the selectors' do + post_validate({ selectors: { items: { selector: 'article' } }, url: 'https://example.com/articles' }) + + json = expect_success_response(last_response) + exported = YAML.safe_load(json.dig('data', 'yaml')) + expect(exported.dig('channel', 'url')).to eq(Html2rss::Web::UrlValidator.canonical_url('https://example.com/articles')) + expect(exported.dig('selectors', 'items', 'selector')).to eq('article') + end + + it 'rejects an unusable page url' do + post_validate({ selectors: { items: { selector: 'article' } }, url: 'not a url' }) + + expect(last_response.status).to eq(400) + expect(JSON.parse(last_response.body).dig('error', 'message')).to eq('Invalid URL format') + end + + it 'returns a parse issue for unparseable yaml' do + post_validate(yaml: "selectors: [\n") + + json = expect_success_response(last_response) + expect(json.dig('data', 'report', 'issues', 0, 'code')).to eq('parse') + end + + it 'rejects channel so the token url cannot be replaced' do + post_validate(yaml: "channel:\n url: https://evil.example\nselectors:\n items:\n selector: article\n") + + expect(last_response.status).to eq(400) + expect(JSON.parse(last_response.body).dig('error', 'message')).to eq('channel is not allowed') + end + end + + describe 'POST /api/v1/feeds/preview', openapi: { + summary: 'Preview a selectors document feed', + description: 'Preview a selectors document feed', + operation_id: 'previewFeed', + tags: ['Studio'], + security: [{ 'BearerAuth' => [] }] + } do + let(:preview_result) do + Html2rss::Test::Result.new( + success: true, item_count: 2, + sample_items: [{ + 'title' => 'A', + 'url' => 'https://example.com/a', + 'published_at' => '2026-01-02T03:04:05Z' + }], + channel_title: 'Example', + channel_url: feed_url, strategy_used: :auto, duration_seconds: 0.1, validation_issues: [], + error_message: nil, failure_kind: nil, rss: 'do-not-leak' + ) + end + let(:failed_preview_result) do + Html2rss::Test::Result.new( + success: false, item_count: 0, sample_items: [], channel_title: nil, + channel_url: feed_url, strategy_used: :auto, duration_seconds: 0.1, validation_issues: [], + error_message: 'no items', failure_kind: Html2rss::Test::FailureKind[:min_items], rss: nil, + quality_report: Html2rss::Test::QualityReport.new( + warnings: [:short_titles], metrics: { 'short_title_count' => 2 }, native_feed: nil, defer_reason: nil + ) + ) + end + + def imaged_preview_result + preview_result.with( + sample_items: [{ 'title' => 'A' }, { 'title' => 'Plain' }, { 'title' => 'Enclosed' }], + rss: %(t#{imaged_items}) + ) + end + + def imaged_items + [ + 'A<img src="https://cdn.example/a.jpg">', + 'Plain', + 'Enclosed' + ].join + end + + def capped_preview_result + rss = %(t#{capped_items}) + preview_result.with(item_count: 12, rss:) + end + + def capped_items + (1..12).map { |index| capped_item(index) }.join + end + + def capped_item(index) + %(Item #{index}https://example.com/#{index}#{capped_image(index)}) + end + + def capped_image(index) + case index + when 1 then '<img src="https://cdn.example/1.jpg">' + when 2 then '<img src="javascript:alert(1)">' + when 3 then '' + when 4 then '' + else '' + end + end + + def expected_capped_rows + images = ['https://cdn.example/1.jpg', nil, nil, 'http://cdn.example/4.jpg', *Array.new(6)] + (1..10).map { |index| ["Item #{index}", images[index - 1]] } + end + + before do + Html2rss::Web::Api::V1::PreviewPageCache.clear! + allow(Html2rss).to receive(:test).and_return(preview_result) + end + + after do + Html2rss::Web::Api::V1::PreviewPageCache.clear! + end + + def post_preview(url) + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url:, selectors: { items: { selector: 'article' } } }.to_json + end + + def preview_strategy + Html2rss::Web::Feeds::GeneratorInput.for_token(url: feed_url, strategy: nil)[:strategy] + end + + def record_preview_results(results) + pending = results.dup + configs = [] + cached_pages = [] + allow(Html2rss).to receive(:test) do |config, **| + path = config.dig(:request, :local_file_path) + cached_pages << (path && File.file?(path) ? File.read(path) : nil) + configs << config + pending.shift || preview_result + end + [configs, cached_pages] + end + + def kept_page_result + preview_result.with(response_body: 'kept') + end + + def rejected_page_results + [ + kept_page_result, + failed_preview_result.with(response_body: 'failed'), + preview_result.with(response_body: ' '), + kept_page_result + ] + end + + it 'selects test fields and never returns the rss body', :aggregate_failures do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + json = expect_success_response(last_response) + expect(json.fetch('data').keys).to contain_exactly( + 'item_count', 'sample_items', 'quality_report', 'failure_kind', 'validation_issues' + ) + expect(json.dig('data', 'item_count')).to eq(2) + end + + it 'reports the failure kind and quality report when extraction comes up empty', :aggregate_failures do + allow(Html2rss).to receive(:test).and_return(failed_preview_result) + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + json = expect_success_response(last_response) + expect(json.dig('data', 'failure_kind')).to eq('min_items') + expect(json.dig('data', 'quality_report', 'warnings')).to eq(['short_titles']) + end + + it 'adds one image url per sample item and still omits the rss document', :aggregate_failures do + allow(Html2rss).to receive(:test).and_return(imaged_preview_result) + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + json = expect_success_response(last_response) + expect(json.dig('data', 'sample_items').map { it['image'] }).to eq( + ['https://cdn.example/a.jpg', nil, 'https://cdn.example/c.jpg'] + ) + expect(json.fetch('data').keys).not_to include('rss') + end + + it 'caps the meadow at ten items and keeps only http(s) images', :aggregate_failures do + allow(Html2rss).to receive(:test).and_return(capped_preview_result) + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + json = expect_success_response(last_response) + samples = json.dig('data', 'sample_items') + expect(json.dig('data', 'item_count')).to eq(12) + expect(samples.map { it.values_at('title', 'image') }).to eq(expected_capped_rows) + end + + it 'keeps gem samples when the rss document cannot be parsed', :aggregate_failures do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + json = expect_success_response(last_response) + expect(json.dig('data', 'sample_items', 0, 'title')).to eq('A') + expect(json.dig('data', 'sample_items', 0)).not_to have_key('image') + end + + it 'replays the same url from the cached page and fetches a different url', :aggregate_failures do + configs, pages = record_preview_results([kept_page_result, kept_page_result, kept_page_result]) + post_preview(feed_url) + post_preview(feed_url) + post_preview('https://other.example/news') + + expect(configs.map { it[:strategy] }).to eq([preview_strategy, :local_file, preview_strategy]) + expect(pages[1]).to eq('kept') + end + + it 'removes the temp file when the cache is cleared' do + configs, = record_preview_results([kept_page_result, kept_page_result]) + post_preview(feed_url) + post_preview(feed_url) + path = configs[1].dig(:request, :local_file_path) + Html2rss::Web::Api::V1::PreviewPageCache.clear! + + expect(File.file?(path)).to be(false) + end + + it 'keeps the stored page when a later body is empty or failed', :aggregate_failures do + configs, pages = record_preview_results(rejected_page_results) + post_preview(feed_url) + post_preview(feed_url) + post_preview('https://other.example/empty') + post_preview(feed_url) + + expect(configs.map { it[:strategy] }).to eq([preview_strategy, :local_file, preview_strategy, :local_file]) + expect(pages.values_at(1, 3)).to eq(['kept', 'kept']) + end + + it 'previews through the token generator config' do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + selectors = Html2rss::Web::SelectorsDocument.from_client(selectors: { items: { selector: 'article' } }) + post '/api/v1/feeds/preview', { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + expect(Html2rss).to have_received(:test).with( + Html2rss::Web::Feeds::GeneratorInput.for_token(url: feed_url, selectors:), + min_items: 1 + ) + end + end + + describe 'POST /api/v1/feeds/suggest_selectors', openapi: { + summary: 'Suggest selectors for a page', + description: 'Suggest selectors for a page', + operation_id: 'suggestSelectors', + tags: ['Studio'], + security: [{ 'BearerAuth' => [] }] + } do + def ranked_candidates + { + items: [ + { selector: 'article', enhance: false, sample: 'First story' }, + { selector: '.story', enhance: true, sample: 'Second story' } + ], + title: [{ selector: 'h2', sample: 'First story' }], + link: [{ selector: 'a', sample: 'First story' }], + published: [{ selector: 'time', sample: 'Yesterday' }] + } + end + + def wire_candidates + { + 'items' => [ + { 'selector' => 'article', 'enhance' => false, 'sample' => 'First story' }, + { 'selector' => '.story', 'enhance' => true, 'sample' => 'Second story' } + ], + 'title' => [{ 'selector' => 'h2', 'sample' => 'First story' }], + 'link' => [{ 'selector' => 'a', 'sample' => 'First story' }], + 'published' => [{ 'selector' => 'time', 'sample' => 'Yesterday' }] + } + end + + def empty_candidates = { items: [], title: [], link: [], published: [] } + + def empty_wire_candidates + { 'items' => [], 'title' => [], 'link' => [], 'published' => [] } + end + + def capture_double(candidates, admission_drops: { 'chrome' => 2 }) + instance_double( + Html2rss::Capture::CaptureResult, + segment_strategy: :list, + admission_drops:, + candidates: + ) + end + + def post_suggest + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post '/api/v1/feeds/suggest_selectors', { url: feed_url, items_selector: 'h2' }.to_json + end + + before { Html2rss::Web::Api::V1::PreviewPageCache.clear! } + after { Html2rss::Web::Api::V1::PreviewPageCache.clear! } + + it 'returns empty candidate buckets as success', :aggregate_failures do + allow(Html2rss).to receive(:capture).and_return(capture_double(empty_candidates, admission_drops: {})) + post_suggest + + json = expect_success_response(last_response) + expect(last_response.status).to eq(200) + expect(json.dig('data', 'candidates')).to eq(empty_wire_candidates) + end + + it 'returns ranked selector candidates', :aggregate_failures do + allow(Html2rss).to receive(:capture).and_return(capture_double(ranked_candidates)) + post_suggest + + json = expect_success_response(last_response) + expect(last_response.status).to eq(200) + expect(json.fetch('data')).to eq( + 'candidates' => wire_candidates, + 'segment_strategy' => 'list', + 'admission_drops' => { 'chrome' => 2 } + ) + end + + it 'replays the cached preview page instead of capturing live', :aggregate_failures do + page = '
Cached
' + result = instance_double(Html2rss::Test::Result, success: true, response_body: page) + Html2rss::Web::Api::V1::PreviewPageCache.store(feed_url, result) + path = Html2rss::Web::Api::V1::PreviewPageCache.path_for(feed_url) + allow(Html2rss).to receive(:capture).and_return(capture_double(empty_candidates, admission_drops: {})) + + post_suggest + + expect(Html2rss).to have_received(:capture).with( + feed_url, + hash_including(strategy: :auto, items_selector: 'h2', local_file_path: path) + ) + expect(path).to be_a(String) + expect(File.read(path)).to eq(page) + end + + it 'captures live when the preview cache has no matching URL', :aggregate_failures do + allow(Html2rss).to receive(:capture).and_return(capture_double(empty_candidates, admission_drops: {})) + + post_suggest + + expect(Html2rss).to have_received(:capture).with( + feed_url, + hash_including(strategy: :auto, items_selector: 'h2') + ) + expect(Html2rss).not_to have_received(:capture).with(anything, hash_including(:local_file_path)) + end + + def raise_empty_capture! + allow(Html2rss).to receive(:capture).and_raise( + Html2rss::NoFeedItemsExtracted.new(attempts: [{ strategy: :default, items_count: 0, error_class: nil }]) + ) + end + + it 'returns EXTRACTION_EMPTY Decision JSON when capture finds no items', :aggregate_failures do + raise_empty_capture! + post_suggest + + expect(last_response.status).to eq(422) + expect(last_response.content_type).to include('application/json') + expect(JSON.parse(last_response.body)).to include( + 'success' => false, + 'error' => include( + 'code' => Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY_CODE, + 'message' => Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY_MESSAGE + ) + ) + end + + it 'returns EXTRACTION_EMPTY JSON in development instead of the exception page', :aggregate_failures do + raise_empty_capture! + + ClimateControl.modify('RACK_ENV' => 'development') do + post_suggest + end + + expect(last_response.status).to eq(422) + expect(last_response.content_type).to include('application/json') + expect(JSON.parse(last_response.body).dig('error', 'code')).to eq( + Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY_CODE + ) + expect(last_response.body).not_to include('NoFeedItemsExtracted') + expect(last_response.body).not_to include('auto_fallback.rb') + end + end + + describe 'studio POSTs with STUDIO_ENABLED=false', openapi: false do + around do |example| + ClimateControl.modify(STUDIO_ENABLED: 'false') { example.run } + end + + Html2rss::Web::Routes::ApiV1::FeedRoutes::STUDIO_POSTS.each_key do |path| + it "returns 403 for POST /api/v1/feeds/#{path}", :aggregate_failures do + header 'Authorization', "Bearer #{admin_token}" + header 'Content-Type', 'application/json' + post "/api/v1/feeds/#{path}", { url: feed_url, selectors: { items: { selector: 'article' } } }.to_json + + expect(last_response.status).to eq(403) + expect(JSON.parse(last_response.body).dig('error', 'message')).to eq('Studio is disabled') + end + end + end end diff --git a/spec/html2rss/web/app_integration_spec.rb b/spec/html2rss/web/app_integration_spec.rb index 8d1717105..28434d3cd 100644 --- a/spec/html2rss/web/app_integration_spec.rb +++ b/spec/html2rss/web/app_integration_spec.rb @@ -231,7 +231,8 @@ def stub_escaped_feed_token(raw_token:, encoded_token:) Html2rss::Web::FeedToken, url: feed_url, username: account[:username], - strategy: 'default' + strategy: 'default', + selectors: nil ) allow(Html2rss::Web::FeedToken::Codec).to receive(:decode).with(raw_token).and_return(escaped_token_payload) @@ -377,12 +378,7 @@ def stub_empty_feed_warning_result end it 'returns corrective extraction-empty failure when extraction is empty', :aggregate_failures do - stub_create_service_result( - status: :empty, - decision: Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY, - cache_key: 'feed_result:create-empty', - empty_reason: 'content_extraction_empty' - ) + stub_empty_extraction_create(cache_key: 'feed_result:create-empty') post '/api/v1/feeds', request_payload.to_json, auth_headers @@ -390,6 +386,19 @@ def stub_empty_feed_warning_result expect(json_body).to include('error' => include(extraction_empty_error_fields)) end + it 'returns EXTRACTION_EMPTY JSON in development instead of the exception page', :aggregate_failures do + stub_empty_extraction_create(cache_key: 'feed_result:create-empty-dev') + + ClimateControl.modify('RACK_ENV' => 'development') do + post '/api/v1/feeds', request_payload.to_json, auth_headers + end + + expect(last_response.status).to eq(422) + expect(last_response.content_type).to include('application/json') + expect(json_body).to include('error' => include(extraction_empty_error_fields)) + expect(last_response.body).not_to include('DecidedError') + end + it 'preserves blocked-surface Decision on create fail-closed', :aggregate_failures do stub_create_service_result( status: :empty, @@ -448,6 +457,17 @@ def extraction_empty_error_fields } end + # @param cache_key [String] + # @return [void] + def stub_empty_extraction_create(cache_key:) + stub_create_service_result( + status: :empty, + decision: Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY, + cache_key:, + empty_reason: 'content_extraction_empty' + ) + end + # @param attrs [Hash{Symbol=>Object}] # @return [void] def stub_create_service_result(**attrs) diff --git a/spec/html2rss/web/feed_token_spec.rb b/spec/html2rss/web/feed_token_spec.rb index 3059af41a..ae669f07d 100644 --- a/spec/html2rss/web/feed_token_spec.rb +++ b/spec/html2rss/web/feed_token_spec.rb @@ -171,6 +171,21 @@ expect(described_class.decode(nil)).to be_nil end + it 'rejects a selectors wire key that is not an object' do + bad_token = Base64.urlsafe_encode64(Zlib::Deflate.deflate({ + p: { u: 'alice', l: 'https://example.com/feed', e: 123, c: 'nope' }, s: 'sig' + }.to_json)) + + expect(described_class.decode(bad_token)).to be_nil + end + + it 'rejects selector wire outside the allowlist' do + wire = { u: 'alice', l: 'https://example.com/feed', e: 123, c: { channel: { url: 'https://evil.example' } } } + bad_token = Base64.urlsafe_encode64(Zlib::Deflate.deflate({ p: wire, s: 'sig' }.to_json)) + + expect(described_class.decode(bad_token)).to be_nil + end + it 'rejects payloads with incorrect types', :aggregate_failures do invalid_payloads = [ { u: 'alice', l: 'https://example.com/feed', e: '123456' }, @@ -203,15 +218,58 @@ end end + describe 'selectors document' do + let(:secret_key) { 'test-secret' } + let(:url) { 'https://example.com/feed' } + let(:selectors) do + Html2rss::Web::SelectorsDocument.from_client(selectors: { items: { selector: 'article', enhance: true } }) + end + + it 'keeps the legacy signature when selectors are absent' do + token = Html2rss::Web::FeedToken::Signer.create(username: 'alice', url:, secret_key:, strategy: 'some_strategy') + legacy = { username: 'alice', url:, expires_at: token.expires_at, strategy: 'some_strategy' } + digest = OpenSSL::HMAC.hexdigest('SHA256', secret_key, JSON.generate(legacy)) + + expect(token.signature).to eq(digest) + end + + it 'round-trips signed selectors and rejects a swap', :aggregate_failures do + token = Html2rss::Web::FeedToken::Signer.create( + username: 'alice', url:, secret_key:, strategy: 'some_strategy', selectors: + ) + encoded = Html2rss::Web::FeedToken::Codec.encode(token) + swapped = Html2rss::Web::SelectorsDocument.from_client(selectors: { items: { selector: 'section' } }) + tampered = Html2rss::Web::FeedToken::Codec.encode(token.with(selectors: swapped)) + + expect(Html2rss::Web::FeedToken::Signer.validate(encoded, url, secret_key).selectors).to eq(selectors) + expect(Html2rss::Web::FeedToken::Signer.validate(tampered, url, secret_key)).to be_nil + end + + it 'signs selectors passed through Auth.generate_feed_token' do + encoded = Html2rss::Web::Auth.generate_feed_token('admin', url, selectors:) + + expect(Html2rss::Web::Auth.validate_and_decode_feed_token(encoded).selectors).to eq(selectors) + end + end + describe '#expired?' do it 'returns true for past timestamps' do - token = described_class.new('alice', 'https://example.com/feed', Time.now.to_i - 1, 'sig', nil) + token = described_class.new( + username: 'alice', + url: 'https://example.com/feed', + expires_at: Time.now.to_i - 1, + signature: 'sig', + strategy: nil + ) expect(token.expired?).to be(true) end it 'returns false for future timestamps' do - token = described_class.new('alice', 'https://example.com/feed', Time.now.to_i + 3600, 'sig', nil) + token = described_class.new( + username: 'alice', url: 'https://example.com/feed', expires_at: Time.now.to_i + 3600, signature: 'sig', + strategy: nil + ) expect(token.expired?).to be(false) end diff --git a/spec/html2rss/web/feeds/generator_input_spec.rb b/spec/html2rss/web/feeds/generator_input_spec.rb new file mode 100644 index 000000000..543f732ef --- /dev/null +++ b/spec/html2rss/web/feeds/generator_input_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Html2rss::Web::Feeds::GeneratorInput do + let(:url) { 'https://example.com/private' } + let(:global_config) do + { + stylesheets: ['/rss.xsl'], + headers: { 'User-Agent' => 'html2rss-web' }, + ignored: 'not copied' + } + end + + before do + allow(Html2rss::Web::LocalConfig).to receive(:global).and_return(global_config) + end + + describe '.for_token' do + it 'reproduces the token generator hash when selectors are absent' do + expect(described_class.for_token(url:, strategy: 'default')).to eq( + stylesheets: ['/rss.xsl'], + headers: { 'User-Agent' => 'html2rss-web' }, + channel: { url: }, + auto_source: {}, + strategy: :default + ) + end + + it 'replaces automatic extraction with signed selectors' do + selectors = Html2rss::Web::SelectorsDocument.from_client( + selectors: { items: { selector: 'article', enhance: true } } + ) + + expect(described_class.for_token(url:, strategy: 'default', selectors:)).to eq( + stylesheets: ['/rss.xsl'], + headers: { 'User-Agent' => 'html2rss-web' }, + channel: { url: }, + strategy: :default, + selectors: { items: { selector: 'article', enhance: true } } + ) + end + + it 'resolves a nil strategy without a caller-supplied name', :aggregate_failures do + allow(Html2rss::Config).to receive(:default_strategy_name).and_return('botasaurus') + + expect(described_class.for_token(url:)[:strategy]).to eq(:botasaurus) + + allow(Html2rss::Config).to receive(:default_strategy_name).and_return(nil) + expect(described_class.for_token(url:, strategy: nil)[:strategy]).to eq(:auto) + end + + it 'rejects an unsupported explicit strategy' do + expect { described_class.for_token(url:, strategy: 'nope') } + .to raise_error(Html2rss::Web::BadRequestError, 'Unsupported strategy') + end + + context 'when serve and preview expand the same token' do + let(:feed_token) do + instance_double(Html2rss::Web::FeedToken, username: 'admin', url:, strategy: nil, selectors: nil) + end + let(:feed_request) do + Html2rss::Web::Feeds::Contracts::Request.new( + target_kind: :token, feed_name: nil, token: 'public-token', params: {} + ) + end + + before do + allow(Html2rss::Web::Auth).to receive(:validate_and_decode_feed_token) + .with('public-token').and_return(feed_token) + allow(Html2rss::Web::AccountManager).to receive(:get_account_by_username) + .with('admin').and_return({ username: 'admin' }) + allow(Html2rss::Web::UrlValidator).to receive(:url_allowed?) + .with({ username: 'admin' }, url).and_return(true) + allow(Html2rss::Web::Flags).to receive(:auto_source_enabled?).and_return(true) + end + + it 'returns one config for both paths' do + served = Html2rss::Web::Feeds::SourceResolver.call(feed_request).generator_input + + expect(served).to eq(described_class.for_token(url:)) + end + + it 'returns one config when both paths carry the same selectors' do + selectors = Html2rss::Web::SelectorsDocument.from_client(selectors: { items: { selector: 'article' } }) + allow(feed_token).to receive(:selectors).and_return(selectors) + served = Html2rss::Web::Feeds::SourceResolver.call(feed_request).generator_input + + expect(served).to eq(described_class.for_token(url:, selectors:)) + end + end + end +end diff --git a/spec/html2rss/web/feeds/source_resolver_spec.rb b/spec/html2rss/web/feeds/source_resolver_spec.rb index d30d882c9..97cfa5fff 100644 --- a/spec/html2rss/web/feeds/source_resolver_spec.rb +++ b/spec/html2rss/web/feeds/source_resolver_spec.rb @@ -89,7 +89,8 @@ def resolved_tuple(resolved) Html2rss::Web::FeedToken, username: 'admin', url: 'https://example.com/private', - strategy: 'default' + strategy: 'default', + selectors: nil ) end @@ -175,6 +176,32 @@ def resolved_tuple(resolved) expect { described_class.call(feed_request) } .to raise_error(Html2rss::Web::ForbiddenError, 'Access Denied') end + + it 're-checks selector wire before building the generator config' do + allow(feed_token).to receive(:selectors).and_return( + instance_double( + Html2rss::Web::SelectorsDocument, + to_wire: { channel: { url: 'https://evil.example' } } + ) + ) + + expect { described_class.call(feed_request) } + .to raise_error(Html2rss::Web::BadRequestError, 'channel is not allowed') + end + + it 'keeps the token url when selectors are signed' do + selectors = Html2rss::Web::SelectorsDocument.from_client( + selectors: { items: { selector: 'article' } } + ) + allow(feed_token).to receive(:selectors).and_return(selectors) + + resolved = described_class.call(feed_request) + + expect(resolved.generator_input).to include( + channel: { url: 'https://example.com/private' }, + selectors: { items: { selector: 'article' } } + ) + end end end end diff --git a/spec/html2rss/web/openapi/document_spec.rb b/spec/html2rss/web/openapi/document_spec.rb new file mode 100644 index 000000000..bba1d140e --- /dev/null +++ b/spec/html2rss/web/openapi/document_spec.rb @@ -0,0 +1,278 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Openapi::Document do + def studio_posts = Html2rss::Web::Routes::ApiV1::FeedRoutes::STUDIO_POSTS + + def get_operation(description: 'ok', summary: nil) + { + get: { + summary: summary || description, + description:, + responses: { '200': { description: } } + } + } + end + + def apply!(paths: nil, components: {}, extra: {}) + spec = { paths: paths || {}, components: { schemas: {} } }.merge(extra) + described_class.apply!(spec, components:) + spec + end + + # rubocop:disable-next Metrics/MethodLength -- dense OpenAPI hull fixture + def studio_hull_spec(nullable_echo: true) + echo = { + type: 'object', + properties: { + items: { + type: 'object', + properties: { selector: { type: 'string' } } + } + } + } + echo[:nullable] = true if nullable_echo + + { + paths: { + '/feeds/preview': { + post: { + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + selectors: { + type: 'object', + properties: { + items: { + type: 'object', + properties: { selector: { type: 'string' } } + } + } + } + } + } + } + } + }, + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + data: { + type: 'object', + properties: { + sample_items: { + type: 'array', + items: { + type: 'object', + properties: { title: { type: 'string' } } + } + }, + candidates: { + type: 'object', + properties: { + items: { type: 'array' }, + title: { type: 'array' }, + link: { type: 'array' }, + published: { type: 'array' } + } + }, + selectors: echo + } + } + } + } + } + } + } + } + } + } + }, + components: { schemas: {} } + } + end + + def issue_hull_spec + properties = %i[actual code expected message path].to_h { [it, { type: 'string' }] } + { + components: { + schemas: { ValidationIssue: { type: 'object', properties: } } + }, + paths: {} + } + end + + describe Openapi::PathTemplate do + [ + ['/api/v1', '/'], + ['/api/v1/feeds', '/feeds'], + ['/api/v1/feeds/validate', '/feeds/validate'], + ['/api/v1/feeds/preview', '/feeds/preview'], + ['/api/v1/feeds/suggest_selectors', '/feeds/suggest_selectors'], + ['/api/v1/feeds/abc123', '/feeds/{token}'], + ['/feeds/{token}', '/feeds/{token}'] + ].each do |raw, expected| + it "maps #{raw} to #{expected}" do + expect(described_class.normalize(raw, studio_posts:)).to eq(expected) + end + end + end + + # rubocop:disable-next RSpec/ExampleLength -- path collapse contract in one example + it 'collapses studio and token paths while keeping symbol keys', :aggregate_failures do + spec = apply!( + paths: { + '/api/v1/feeds/validate': get_operation(description: 'validate'), + '/api/v1/feeds/preview': get_operation(description: 'preview'), + '/api/v1/feeds/suggest_selectors': get_operation(description: 'suggest'), + '/api/v1/feeds/abc123': get_operation(description: 'token a'), + '/api/v1/feeds/def456': get_operation(description: 'token a') + } + ) + + expect(spec[:paths].keys).to contain_exactly( + :'/feeds/validate', :'/feeds/preview', :'/feeds/suggest_selectors', :'/feeds/{token}' + ) + expect(spec[:paths].keys).to all(be_a(Symbol)) + end + + it 'merges token operations and inserts the path parameter', :aggregate_failures do + spec = apply!( + paths: { + '/api/v1/feeds/one': get_operation(description: 'same'), + '/api/v1/feeds/two': get_operation(description: 'same') + } + ) + get = spec.dig(:paths, :'/feeds/{token}', :get) + + expect(get[:parameters]).to include( + name: 'token', in: 'path', required: true, schema: { type: 'string' } + ) + expect(get.dig(:responses, :'200', :description)).to eq('same') + end + + it 'raises when merged operations contribute conflicting descriptions' do + expect do + apply!( + paths: { + '/api/v1/feeds/one': get_operation(description: 'alpha'), + '/api/v1/feeds/two': get_operation(description: 'beta') + } + ) + end.to raise_error(Openapi::Document::ConflictError, /alpha | beta|beta | alpha/) + end + + it 'stamps issue code enums and open expected/actual schemas', :aggregate_failures do + spec = issue_hull_spec + described_class.apply!(spec, components: {}) + + code = spec.dig(:components, :schemas, :ValidationIssue, :properties, :code) + expect(code[:enum]).to eq(Html2rss::Config::ValidationIssue::CODES.map(&:to_s).sort) + expect(code[:type]).to eq('string') + + %i[expected actual].each do |field| + expect(spec.dig(:components, :schemas, :ValidationIssue, :properties, field)).to eq( + description: Openapi::Document::OPEN_VALUE_DESCRIPTION + ) + end + end + + it 'publishes web schemas with 3.1 null sample fields and no nullable keys', :aggregate_failures do + spec = apply! + sample = spec.dig(:components, :schemas, :PreviewSampleItem) + + expect(sample[:required]).to eq(%w[title]) + expect(sample[:properties][:title]).to eq(type: 'string') + expect(sample[:properties][:url]).to eq(type: %w[string null]) + expect(sample[:properties][:image]).to eq(type: %w[string null]) + expect(spec.to_s).not_to include('nullable') + + expect(spec.dig(:components, :schemas).keys).to include( + :PreviewSampleItem, :SelectorCandidates, :ItemsSelectorCandidate, :FieldSelectorCandidate + ) + expect(spec.dig(:components, :schemas)).not_to have_key(:SelectorsDocument) + end + + it 'merges JsonSchema.components when components are omitted', :aggregate_failures do + spec = studio_hull_spec + described_class.apply!(spec) + + expect(spec.dig(:components, :schemas)).to have_key(:SelectorsDocument) + request_selectors = spec.dig( + :paths, :'/feeds/preview', :post, :requestBody, :content, :'application/json', + :schema, :properties, :selectors + ) + expect(request_selectors).to eq('$ref': '#/components/schemas/SelectorsDocument') + end + + # rubocop:disable-next RSpec/ExampleLength -- hull leave-alone contract in one example + it 'leaves selector hulls alone when SelectorsDocument is absent', :aggregate_failures do + spec = studio_hull_spec + described_class.apply!(spec, components: {}) + + request_selectors = spec.dig( + :paths, :'/feeds/preview', :post, :requestBody, :content, :'application/json', + :schema, :properties, :selectors + ) + expect(request_selectors[:type]).to eq('object') + expect(request_selectors).not_to have_key(:$ref) + + sample_items = spec.dig( + :paths, :'/feeds/preview', :post, :responses, :'200', :content, :'application/json', + :schema, :properties, :data, :properties, :sample_items + ) + expect(sample_items).to eq(type: 'array', items: { '$ref': '#/components/schemas/PreviewSampleItem' }) + end + + # rubocop:disable-next RSpec/ExampleLength -- hull rewrite + oneOf contract in one example + it 'rewrites selector hulls to refs and nullable echoes to oneOf when present', :aggregate_failures do + spec = studio_hull_spec + described_class.apply!( + spec, + components: { 'SelectorsDocument' => { 'type' => 'object' } } + ) + + request_selectors = spec.dig( + :paths, :'/feeds/preview', :post, :requestBody, :content, :'application/json', + :schema, :properties, :selectors + ) + expect(request_selectors).to eq('$ref': '#/components/schemas/SelectorsDocument') + + echo = spec.dig( + :paths, :'/feeds/preview', :post, :responses, :'200', :content, :'application/json', + :schema, :properties, :data, :properties, :selectors + ) + expect(echo).to eq( + oneOf: [ + { type: 'null' }, + { '$ref': '#/components/schemas/SelectorsDocument' } + ] + ) + + candidates = spec.dig( + :paths, :'/feeds/preview', :post, :responses, :'200', :content, :'application/json', + :schema, :properties, :data, :properties, :candidates + ) + expect(candidates).to eq('$ref': '#/components/schemas/SelectorCandidates') + expect(spec.to_s).not_to include('nullable') + end + + it 'deep-sorts keys and keeps top-level keys as symbols', :aggregate_failures do + spec = apply!( + paths: { '/api/v1/feeds': get_operation(description: 'feeds') }, + extra: { openapi: '3.1.0', info: { title: 't', z: 1, a: 2 } } + ) + + expect(spec.keys).to eq(%i[components info openapi paths]) + expect(spec[:info].keys).to eq(%i[a title z]) + expect(spec[:paths].keys).to all(be_a(Symbol)) + end +end diff --git a/spec/html2rss/web/openapi/json_schema_spec.rb b/spec/html2rss/web/openapi/json_schema_spec.rb new file mode 100644 index 000000000..65dd1e4d9 --- /dev/null +++ b/spec/html2rss/web/openapi/json_schema_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require_relative '../../../support/openapi/json_schema' + +RSpec.describe Openapi::JsonSchema do + describe '.components' do + subject(:components) { described_class.components } + + # rubocop:disable-next RSpec/ExampleLength -- one mounted-schema contract example + it 'translates the mounted gem selectors schema into OpenAPI 3.1 components', + :aggregate_failures do + root = components.fetch('SelectorsDocument') + denied = %w[channel request strategy headers] + dump = JSON.generate(components) + + denied.each do |name| + expect(components).not_to have_key(name) + expect(root.fetch('properties')).not_to have_key(name) + end + + expect(root).to have_key('patternProperties') + expect(components.dig('Html2rssPostProcessorGsub', 'properties', 'name')).to include( + 'const' => 'gsub' + ) + expect(components.dig('Html2rssPostProcessorGsub', 'properties', 'name')).not_to have_key('enum') + expect(root.dig('properties', 'items', 'properties', 'order')).to have_key('not') + expect( + root.dig('properties', 'items', 'properties', 'pagination', 'oneOf', 0, 'exclusiveMinimum') + ).to eq(0) + expect(dump).not_to include('#/$defs/post_processors/gsub') + expect(dump).to include('#/components/schemas/Html2rssPostProcessorGsub') + expect(dump).not_to include('nullable') + end + + it 'preserves a type union that includes null as an array', :aggregate_failures do + title = { + 'type' => 'object', + 'properties' => { + 'selectors' => { + 'type' => 'object', + 'properties' => { 'title' => { 'type' => %w[string null] } } + } + }, + '$defs' => {} + }.then { described_class.components(it).dig('SelectorsDocument', 'properties', 'title') } + + expect(title).to eq('type' => %w[string null]) + expect(JSON.generate(title)).not_to include('nullable') + end + + [ + ['unknown keyword', { 'type' => 'object', 'mystery' => true }, %r{/properties/selectors/mystery}], + [ + 'unknown $ref target shape', + { 'type' => 'object', '$ref' => '#/definitions/other' }, + %r{/properties/selectors/\$ref} + ] + ].each do |label, selectors, pointer| + it "raises Unsupported for #{label}" do + document = { + 'type' => 'object', + 'properties' => { 'selectors' => selectors }, + '$defs' => {} + } + + expect { described_class.components(document) } + .to raise_error(Openapi::JsonSchema::Unsupported, pointer) + end + end + end +end diff --git a/spec/html2rss/web/openapi/selectors_contract_spec.rb b/spec/html2rss/web/openapi/selectors_contract_spec.rb new file mode 100644 index 000000000..0eb2a37ee --- /dev/null +++ b/spec/html2rss/web/openapi/selectors_contract_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'SelectorsDocument OpenAPI contract' do + describe 'accept fixtures' do + SelectorsContractFixtures::ACCEPT.each do |label, selectors| + it "accepts #{label} via SelectorsDocument.from_client" do + document = Html2rss::Web::SelectorsDocument.from_client(selectors:) + + expect(document.selectors).to eq(selectors) + end + end + end + + describe 'reject fixtures' do + SelectorsContractFixtures::REJECT.each do |label, fragment| + it "rejects #{label} with BadRequestError" do + expect { Html2rss::Web::SelectorsDocument.from_client(fragment) } + .to raise_error(Html2rss::Web::BadRequestError) + end + end + end + + describe 'OpenAPI mount' do + subject(:root) { Openapi::JsonSchema.components.fetch('SelectorsDocument') } + + it 'publishes items/order/pagination, keeps patternProperties, omits denied roots', + :aggregate_failures do + properties = root.fetch('properties') + items_properties = properties.dig('items', 'properties') + denied = Html2rss::Web::SelectorsDocument::DENIED_ROOT_KEYS.map(&:to_s) + + expect(properties).to have_key('items') + expect(items_properties).to include('order', 'pagination') + expect(root).to have_key('patternProperties') + denied.each { expect(properties).not_to have_key(it) } + end + end +end diff --git a/spec/html2rss/web/rate_limiter_spec.rb b/spec/html2rss/web/rate_limiter_spec.rb index f7cffc87a..fde24c377 100644 --- a/spec/html2rss/web/rate_limiter_spec.rb +++ b/spec/html2rss/web/rate_limiter_spec.rb @@ -38,6 +38,60 @@ expect(body.dig('error', 'code')).to eq('TOO_MANY_REQUESTS') end + %w[preview suggest_selectors].each do |endpoint| + it "weights #{endpoint} and keeps its 429 response API-shaped", :aggregate_failures do + allow(Html2rss::Web::Flags).to receive(:rate_limit_max_requests).and_return(10) + + 2.times do + expect(request_builder.post("/api/v1/feeds/#{endpoint}").status).to eq(200) + end + + response = request_builder.post("/api/v1/feeds/#{endpoint}") + expect(response.status).to eq(429) + expect(response['Content-Type']).to eq('application/json') + expect(JSON.parse(response.body).dig('error', 'code')).to eq('TOO_MANY_REQUESTS') + end + end + + it 'weights feed creation as a live fetch', :aggregate_failures do + allow(Html2rss::Web::Flags).to receive(:rate_limit_max_requests).and_return(10) + + 2.times do + expect(request_builder.post('/api/v1/feeds').status).to eq(200) + end + + response = request_builder.post('/api/v1/feeds') + expect(response.status).to eq(429) + expect(response['Content-Type']).to eq('application/json') + end + + it 'keeps a new studio POST API-shaped on 429', :aggregate_failures do + stub_const( + 'Html2rss::Web::Routes::ApiV1::FeedRoutes::STUDIO_POSTS', + { 'future_tool' => Html2rss::Web::Api::V1::PreviewFeed }.freeze + ) + allow(Html2rss::Web::Flags).to receive(:rate_limit_max_requests).and_return(1) + builder = Rack::MockRequest.new(described_class.new(inner_app)) + + expect(builder.post('/api/v1/feeds/future_tool').status).to eq(200) + + response = builder.post('/api/v1/feeds/future_tool') + expect(response.status).to eq(429) + expect(response['Content-Type']).to eq('application/json') + expect(JSON.parse(response.body).dig('error', 'code')).to eq('TOO_MANY_REQUESTS') + end + + it 'keeps a feed path outside the studio post set feed-shaped on 429', :aggregate_failures do + allow(Html2rss::Web::Flags).to receive(:rate_limit_max_requests).and_return(1) + builder = Rack::MockRequest.new(described_class.new(inner_app)) + + builder.get('/api/v1/feeds/not-a-studio-post') + response = builder.get('/api/v1/feeds/not-a-studio-post') + + expect(response.status).to eq(429) + expect(response['Content-Type']).to eq(Html2rss::Web::Feeds::FormatNegotiation::TEXT_PLAIN_CONTENT_TYPE) + end + it 'bypasses rate limiting for health check routes' do 4.times do response = request_builder.get('/api/v1/health') diff --git a/spec/html2rss/web/security/selectors_document_spec.rb b/spec/html2rss/web/security/selectors_document_spec.rb new file mode 100644 index 000000000..fd4959fa9 --- /dev/null +++ b/spec/html2rss/web/security/selectors_document_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Html2rss::Web::SelectorsDocument do # rubocop:disable RSpec/SpecFilePathFormat -- lives under security/ with feed token + let(:selectors) { { items: { selector: 'article', enhance: true } } } + let(:fragment) { { selectors: } } + + # @param pattern [String] + # @return [Hash{Symbol=>Object}] + def selectors_with_gsub(pattern) + { + items: { selector: 'article' }, + title: { selector: 'h2', post_process: [{ name: 'gsub', pattern:, replacement: ' ' }] } + } + end + + describe '.from_client and .from_wire' do + it 'accepts a selectors document fragment and freezes the handoff', :aggregate_failures do + document = described_class.from_client(fragment) + + expect(document.to_config_fragment).to eq(selectors:) + expect(document.to_wire).to eq(document.to_config_fragment) + expect(document).not_to be_empty + expect { document.selectors[:items][:selector] << 'x' }.to raise_error(FrozenError) + expect(described_class.from_wire(document.to_wire)).to eq(document) + end + + it 'does not freeze the caller hash' do + raw = { 'selectors' => { 'items' => { 'selector' => +'article', 'enhance' => true } } } + + described_class.from_client(raw) + + expect { raw['selectors']['items']['selector'] << 'x' }.not_to raise_error + end + + it 'accepts string keys' do + document = described_class.from_client( + 'selectors' => { 'items' => { 'selector' => 'article', 'enhance' => false } } + ) + + expect(document.selectors).to eq(items: { selector: 'article', enhance: false }) + end + + described_class::DENIED_ROOT_KEYS.each do |key| + it "rejects #{key} on mint and on decode", :aggregate_failures do + payload = fragment.merge(key => { 'url' => 'https://evil.example' }) + + expect { described_class.from_client(payload) } + .to raise_error(Html2rss::Web::BadRequestError, "#{key} is not allowed") + expect { described_class.from_wire(payload) } + .to raise_error(Html2rss::Web::BadRequestError, "#{key} is not allowed") + end + end + + it 'rejects unknown root keys' do + expect { described_class.from_client(foo: 1) } + .to raise_error(Html2rss::Web::BadRequestError, 'foo is not allowed') + end + + it 'rejects a non-object fragment' do + expect { described_class.from_wire('selectors') } + .to raise_error(Html2rss::Web::BadRequestError, 'selectors document must be an object') + end + + it 'rejects a non-object selectors value' do + expect { described_class.from_client(selectors: 'article') } + .to raise_error(Html2rss::Web::BadRequestError, 'selectors must be an object') + end + + it 'rejects fragments over the wire byte cap' do + payload = { selectors: { items: { selector: 'a' * (described_class::MAX_WIRE_BYTES + 1) } } } + + expect { described_class.from_client(payload) }.to raise_error( + Html2rss::Web::BadRequestError, + "selectors document exceeds #{described_class::MAX_WIRE_BYTES} bytes" + ) + end + + it 'delegates shape failures to config validation' do + expect { described_class.from_client(selectors: { items: { selector: '' } }) } + .to raise_error(Html2rss::Web::BadRequestError, /selectors\.items\.selector \[invalid_value\]/) + end + end + + describe 'gsub post-processors' do + it 'admits a gsub pattern and carries it into the signed wire', :aggregate_failures do + document = described_class.from_client(selectors: selectors_with_gsub('\\s+')) + encoded = Html2rss::Web::Auth.generate_feed_token('admin', 'https://example.com/feed', selectors: document) + + expect(document.to_wire.dig(:selectors, :title, :post_process, 0, :pattern)).to eq('\\s+') + expect(Html2rss::Web::Auth.validate_and_decode_feed_token(encoded).selectors).to eq(document) + end + + it 'leaves the pattern bounds to the gem compile step', :aggregate_failures do + expect { described_class.from_client(selectors: selectors_with_gsub('(a+)+$')) } + .to raise_error(Html2rss::Web::BadRequestError, /pattern contains nested quantifiers/) + expect { described_class.from_client(selectors: selectors_with_gsub('a' * 300)) } + .to raise_error(Html2rss::Web::BadRequestError, /pattern exceeds 256 characters/) + end + + it 'still rejects a denied root key that travels with a post_process' do + payload = { channel: { url: 'https://evil.example' }, selectors: selectors_with_gsub('x') } + + expect { described_class.from_client(payload) } + .to raise_error(Html2rss::Web::BadRequestError, 'channel is not allowed') + end + end +end diff --git a/spec/support/openapi.rb b/spec/support/openapi.rb index 11c3188c9..1ae9cf90e 100644 --- a/spec/support/openapi.rb +++ b/spec/support/openapi.rb @@ -6,13 +6,12 @@ RSpec::OpenAPI.path = 'public/openapi.yaml' RSpec::OpenAPI.title = 'html2rss-web API' RSpec::OpenAPI.application_version = Html2rss::Web::VERSION - # Spectral `oas3-schema` validates OAS 3.0; pin so nullables use `nullable: true` - # instead of OAS 3.1+ `type: [..., null]` unions. - RSpec::OpenAPI.openapi_version = '3.0.3' + RSpec::OpenAPI.openapi_version = '3.1.0' RSpec::OpenAPI.enable_example = false RSpec::OpenAPI.enable_example_summary = false RSpec::OpenAPI.example_types = [:request] RSpec::OpenAPI.request_headers = ['Authorization'] + RSpec::OpenAPI.response_headers = %w[Retry-After Vary Link Cache-Control] RSpec::OpenAPI.servers = [ { url: 'https://api.html2rss.dev/api/v1', description: 'Production server' }, { url: 'http://127.0.0.1:4000/api/v1', description: 'Development server' } @@ -36,205 +35,17 @@ bearerFormat: 'JWT' } } - - RSpec::OpenAPI.summary_builder = lambda { |example| - example.metadata.dig(:example_group, :openapi, :summary) || example.metadata[:summary] - } - RSpec::OpenAPI.tags_builder = lambda { |example| - example.metadata.dig(:example_group, :openapi, :tags) || example.metadata[:tags] - } - RSpec::OpenAPI.description_builder = lambda { |example| - example.metadata.dig(:example_group, :openapi, - :description) || example.metadata[:description] || example.description - } + RSpec::OpenAPI.root_tags = [ + { name: 'Root', description: 'API metadata and service-level information.' }, + { name: 'Catalog', description: 'Public feed-directory catalog metadata.' }, + { name: 'Health', description: 'Health and readiness endpoints.' }, + { name: 'Strategies', description: 'Feed extraction strategy discovery.' }, + { name: 'Feeds', description: 'Feed creation and feed rendering operations.' }, + { name: 'Studio', description: 'Validate, preview, and suggest refined feed configs.' } + ] # Keep path keys relative to /api/v1 because servers include the versioned base path. RSpec::OpenAPI.post_process_hook = lambda do |_path, _records, spec| - token_feed_error_statuses = %w[401 403 429 500 503 504].freeze - token_feed_empty_statuses = %w[422].freeze - - stringify = lambda do |value| - case value - when Hash - value.each_with_object({}) { |(key, nested_value), mapped| mapped[key.to_s] = stringify.call(nested_value) } - when Array - value.map { |item| stringify.call(item) } - else - value - end - end - - deep_sort = lambda do |value| - case value - when Hash - value.keys.sort_by(&:to_s).to_h { |key| [key, deep_sort.call(value[key])] } - when Array - value.map { |item| deep_sort.call(item) } - else - value - end - end - - merge_responses = lambda do |existing_responses, new_responses| - canonical_description = lambda do |*responses| - descriptions = responses - .filter_map { |response| response['description']&.to_s&.strip } - .reject(&:empty?) - .uniq - - next nil if descriptions.empty? - - # Prefer the most generic/canonical wording when duplicate examples define - # the same status differently. - descriptions.min_by { |description| [description.length, description] } - end - - statuses = existing_responses.keys | new_responses.keys - - statuses.each_with_object({}) do |status, merged_responses| - current = existing_responses[status] || {} - incoming = new_responses[status] || {} - merged_response = current.merge(incoming) - - current_content = current['content'] || {} - incoming_content = incoming['content'] || {} - if current_content.any? || incoming_content.any? - content_types = current_content.keys | incoming_content.keys - merged_response['content'] = content_types.to_h do |content_type| - current_entry = current_content[content_type] || {} - incoming_entry = incoming_content[content_type] || {} - [content_type, current_entry.merge(incoming_entry)] - end - end - - current_headers = current['headers'] || {} - incoming_headers = incoming['headers'] || {} - if current_headers.any? || incoming_headers.any? - merged_response['headers'] = current_headers.merge(incoming_headers) - end - - merged_response['description'] = canonical_description.call(current, incoming) - merged_responses[status] = merged_response - end - end - - token_feed_error_examples = { - 'text/plain' => { - 'example' => 'Failed to generate feed: Invalid token' - } - } - - token_feed_empty_examples = { - 'text/plain' => { - 'example' => Html2rss::Web::ErrorClassifier::EXTRACTION_EMPTY_MESSAGE - } - } - - token_feed_error_content = { - 'text/plain' => { - 'schema' => { 'type' => 'string' } - }.merge(token_feed_error_examples.fetch('text/plain')) - }.freeze - token_feed_empty_content = { - 'text/plain' => { - 'schema' => { 'type' => 'string' } - }.merge(token_feed_empty_examples.fetch('text/plain')) - }.freeze - - path_map = spec['paths'] || spec[:paths] - next unless path_map.is_a?(Hash) - - normalized_paths = {} - path_map.each do |raw_path, operation| - original_path = raw_path.to_s - normalized = if original_path.match?(%r{\A/api/v1/feeds/[^/]+\z}) - '/feeds/{token}' - elsif original_path.start_with?('/api/v1') - original_path.delete_prefix('/api/v1') - else - original_path - end - normalized = '/' if normalized.empty? - normalized_paths[normalized] ||= {} - - stringify.call(operation).each do |verb, operation_doc| - existing = normalized_paths[normalized][verb] - - if existing - merged = existing.merge(operation_doc) - merged['responses'] = merge_responses.call(existing['responses'] || {}, operation_doc['responses'] || {}) - merged['parameters'] = [*(existing['parameters'] || []), *(operation_doc['parameters'] || [])] - merged['parameters'].uniq! { |parameter| [parameter['name'], parameter['in']] } - normalized_paths[normalized][verb] = deep_sort.call(merged) - else - normalized_paths[normalized][verb] = deep_sort.call(operation_doc) - end - - normalized_paths[normalized][verb]['description'] ||= normalized_paths[normalized][verb]['summary'] - - if normalized.start_with?('/feeds/{token}') - normalized_paths[normalized][verb]['parameters'] ||= [] - has_token_param = normalized_paths[normalized][verb]['parameters'].any? do |parameter| - parameter['name'] == 'token' && parameter['in'] == 'path' - end - unless has_token_param - normalized_paths[normalized][verb]['parameters'] << { - 'name' => 'token', - 'in' => 'path', - 'required' => true, - 'schema' => { 'type' => 'string' } - } - end - end - - # Inject Retry-After header for rate limiting and timeout responses - %w[429 503 504].each do |status| # rubocop:disable Performance/CollectionLiteralInLoop - response = normalized_paths[normalized][verb].dig('responses', status) - next unless response - - response['headers'] ||= {} - response['headers']['Retry-After'] = { - 'description' => 'The number of seconds to wait before retrying the request.', - 'schema' => { 'type' => 'integer' } - } - end - - next unless normalized == '/feeds/{token}' - - token_feed_error_statuses.each do |status| - response = normalized_paths[normalized][verb].dig('responses', status) - next unless response - - response['content'] = token_feed_error_content - end - - token_feed_empty_statuses.each do |status| - response = normalized_paths[normalized][verb].dig('responses', status) - next unless response - - response['content'] = token_feed_empty_content - end - end - end - - if spec.key?('paths') - spec['paths'] = deep_sort.call(normalized_paths) - else - spec[:paths] = deep_sort.call(normalized_paths) - end - - tags = [ - { 'name' => 'Root', 'description' => 'API metadata and service-level information.' }, - { 'name' => 'Catalog', 'description' => 'Public feed-directory catalog metadata.' }, - { 'name' => 'Health', 'description' => 'Health and readiness endpoints.' }, - { 'name' => 'Strategies', 'description' => 'Feed extraction strategy discovery.' }, - { 'name' => 'Feeds', 'description' => 'Feed creation and feed rendering operations.' } - ] - - if spec.key?('tags') - spec['tags'] = tags - else - spec[:tags] = tags - end + Openapi::Document.apply!(spec) end end diff --git a/spec/support/openapi/document.rb b/spec/support/openapi/document.rb new file mode 100644 index 000000000..417cf5d93 --- /dev/null +++ b/spec/support/openapi/document.rb @@ -0,0 +1,300 @@ +# frozen_string_literal: true + +## +# Typed OpenAPI document helpers for the rspec-openapi post-process hook. +module Openapi + ## + # Collapses recorded request paths to server-relative OpenAPI templates. + module PathTemplate + module_function + + # @param path [String, Symbol] + # @param studio_posts [Hash, #keys] studio path segment → handler map + # @return [String] + # rubocop:disable-next Metrics/MethodLength -- path collapse branches stay local + def normalize(path, studio_posts:) + studio = Regexp.union(Array(studio_posts.keys).map(&:to_s)).source + token_path = %r{\A/api/v1/feeds/(?!(?:#{studio})\z)[^/]+\z} + original = path.to_s + normalized = if original.match?(token_path) + '/feeds/{token}' + elsif original.start_with?('/api/v1') + original.delete_prefix('/api/v1') + else + original + end + normalized.empty? ? '/' : normalized + end + end + + ## + # Mutates the symbol-keyed OpenAPI document: path merge, web stamps, hull refs. + module Document # rubocop:disable Metrics/ModuleLength -- one deep generation seam + class ConflictError < StandardError + end + + SELECTORS_DOCUMENT_REF = { '$ref': '#/components/schemas/SelectorsDocument' }.freeze + PREVIEW_SAMPLE_ITEM_REF = { '$ref': '#/components/schemas/PreviewSampleItem' }.freeze + SELECTOR_CANDIDATES_REF = { '$ref': '#/components/schemas/SelectorCandidates' }.freeze + ITEMS_CANDIDATE_REF = { '$ref': '#/components/schemas/ItemsSelectorCandidate' }.freeze + FIELD_CANDIDATE_REF = { '$ref': '#/components/schemas/FieldSelectorCandidate' }.freeze + OPEN_VALUE_DESCRIPTION = 'JSON-ish value, or null when the issue carries none.' + + # rubocop:disable-next Metrics/ClassLength -- generation hook owns path merge + stamps + class << self + # @param spec [Hash] symbol-keyed OpenAPI document + # @param components [Hash] schema name → schema; defaults to {Openapi::JsonSchema.components} + # @return [void] + # rubocop:disable-next Metrics/MethodLength -- ordered mutate steps for one document + def apply!(spec, components: JsonSchema.components) + path_map = spec[:paths] + if path_map.is_a?(Hash) + studio_posts = Html2rss::Web::Routes::ApiV1::FeedRoutes::STUDIO_POSTS + spec[:paths] = normalize_paths(path_map, studio_posts:) + end + + walk(spec) { |node| stamp_issue_schema(node) } + schemas = ((spec[:components] ||= {})[:schemas] ||= {}) + publish_web_schemas(schemas) + merge_components(schemas, components) + rewrite_hulls(spec, selectors: schemas.key?(:SelectorsDocument)) + spec.replace(deep_sort(spec)) + end + + private + + def deep_sort(value) + case value + in Hash then value.keys.sort_by(&:to_s).to_h { |key| [key, deep_sort(value[key])] } + in Array then value.map { deep_sort(it) } + else value + end + end + + def normalize_paths(path_map, studio_posts:) + path_map.each_with_object({}) do |(raw, operations), paths| + add_operations(paths, PathTemplate.normalize(raw, studio_posts:), operations) + end + end + + def add_operations(paths, normalized, operations) + key = normalized.to_sym + bucket = paths[key] ||= {} + operations.each do |verb, doc| + bucket[verb] = merge_operation(bucket[verb], doc) + decorate_operation(bucket[verb], normalized) + end + end + + def merge_operation(existing, operation_doc) + return deep_sort(operation_doc) unless existing + + raise_description_conflict!(existing, operation_doc) + merged = existing.merge(operation_doc) + merged[:responses] = merge_responses(existing[:responses] || {}, operation_doc[:responses] || {}) + merged[:parameters] = [*(existing[:parameters] || []), *(operation_doc[:parameters] || [])] + merged[:parameters].uniq! { |parameter| [parameter[:name], parameter[:in]] } + deep_sort(merged) + end + + def decorate_operation(doc, normalized) + doc[:description] ||= doc[:summary] + ensure_token_parameter(doc) if normalized.include?('{token}') + end + + def ensure_token_parameter(doc) + params = doc[:parameters] ||= [] + return if params.any? { it[:name] == 'token' && it[:in] == 'path' } + + params << { name: 'token', in: 'path', required: true, schema: { type: 'string' } } + end + + def merge_responses(existing_responses, new_responses) + (existing_responses.keys | new_responses.keys).each_with_object({}) do |status, merged| + current = existing_responses[status] || {} + incoming = new_responses[status] || {} + merged[status] = current.merge(incoming).tap do |response| + copy_merged_content(response, current, incoming) + copy_merged_headers(response, current, incoming) + response[:description] = merged_description(current, incoming) + end + end + end + + def copy_merged_content(response, current, incoming) + left = Hash(current[:content]) + right = Hash(incoming[:content]) + types = left.keys | right.keys + return if types.empty? + + response[:content] = types.to_h { |type| [type, left.fetch(type, {}).merge(right.fetch(type, {}))] } + end + + def copy_merged_headers(response, current, incoming) + left = Hash(current[:headers]) + right = Hash(incoming[:headers]) + return if left.empty? && right.empty? + + response[:headers] = left.merge(right) + end + + def raise_description_conflict!(left, right) = merged_description(left, right) + + def merged_description(*nodes) + descriptions = nodes.filter_map { it[:description]&.to_s&.strip }.reject(&:empty?).uniq + case descriptions + in [] then nil + in [one] then one + else + raise ConflictError, "conflicting OpenAPI descriptions: #{descriptions.join(' | ')}" + end + end + + def walk(node, &) + case node + in Hash + yield node + node.each_value { walk(it, &) } + in Array + node.each { walk(it, &) } + else + nil + end + end + + def stamp_issue_schema(node) + properties = node[:properties] + return unless properties.is_a?(Hash) && issue_schema?(properties) + + stamp_issue_code(properties[:code]) + stamp_open_value(properties[:expected]) + stamp_open_value(properties[:actual]) + end + + def issue_schema?(properties) = %i[actual code expected message path].all? { properties.key?(it) } + + def stamp_issue_code(code) + return unless code.is_a?(Hash) + + code[:type] = 'string' + code[:enum] = Html2rss::Config::ValidationIssue::CODES.map(&:to_s).sort + end + + def stamp_open_value(value) + return unless value.is_a?(Hash) + + value.replace(description: OPEN_VALUE_DESCRIPTION) + end + + # rubocop:disable-next Metrics/MethodLength -- four web component schemas in one stamp + def publish_web_schemas(schemas) + names = bucket_names + schemas[:PreviewSampleItem] = { + description: 'One preview meadow row returned by POST /feeds/preview.', + type: 'object', required: %w[title], + properties: Html2rss::Web::Api::V1::PreviewSamples::SAMPLE_KEYS.to_h do |key| + [key.to_sym, key == 'title' ? { type: 'string' } : { type: %w[string null] }] + end + } + schemas[:SelectorCandidates] = { + description: 'Ranked selector candidates. Empty buckets are a successful response.', + type: 'object', required: names.map(&:to_s), additionalProperties: false, + properties: names.to_h do |name| + ref = name == :items ? ITEMS_CANDIDATE_REF : FIELD_CANDIDATE_REF + [name, { type: 'array', items: ref.dup }] + end + } + schemas[:ItemsSelectorCandidate] = { + description: 'One ranked items selector with a sample headline. Best match is first.', + type: 'object', required: %w[selector enhance sample], additionalProperties: false, + properties: { + selector: { type: 'string' }, + enhance: { type: 'boolean' }, + sample: { type: 'string', description: 'Visible text of a matched node' } + } + } + schemas[:FieldSelectorCandidate] = { + description: 'One ranked field selector with a sample headline. Best match is first.', + type: 'object', required: %w[selector sample], additionalProperties: false, + properties: { + selector: { type: 'string' }, + sample: { type: 'string', description: 'Visible text of a matched node' } + } + } + end + + def merge_components(schemas, components) + return if components.nil? || components.empty? + + symbolize(components).each { |name, schema| schemas[name] = schema } + end + + def symbolize(value) + require 'rspec/openapi' unless defined?(RSpec::OpenAPI::KeyTransformer) + + RSpec::OpenAPI::KeyTransformer.symbolize(value) + end + + def bucket_names = Html2rss::Web::Api::V1::SuggestSelectors::BUCKET_NAMES.map { it.to_s.to_sym } + + def rewrite_hulls(spec, selectors:) + walk(spec) do |node| + properties = node[:properties] + next unless properties.is_a?(Hash) + + rewrite_selectors_property(properties) if selectors + rewrite_sample_items_property(properties) + rewrite_candidates_property(properties) + end + end + + def rewrite_selectors_property(properties) + schema = properties[:selectors] + return unless selectors_document_hull?(schema) + + properties[:selectors] = if nullable_schema?(schema) + { oneOf: [{ type: 'null' }, SELECTORS_DOCUMENT_REF.dup] } + else + SELECTORS_DOCUMENT_REF.dup + end + end + + def nullable_schema?(schema) + schema[:nullable] || (schema[:type].is_a?(Array) && schema[:type].include?('null')) + end + + def selectors_document_hull?(schema) + return false unless schema.is_a?(Hash) && !schema.key?(:$ref) + + items = schema.dig(:properties, :items) + items.is_a?(Hash) && items.dig(:properties, :selector).is_a?(Hash) + end + + def rewrite_sample_items_property(properties) + schema = properties[:sample_items] + return unless sample_items_hull?(schema) + + properties[:sample_items] = { type: 'array', items: PREVIEW_SAMPLE_ITEM_REF.dup } + end + + def sample_items_hull?(schema) + schema.is_a?(Hash) && schema[:type] == 'array' && !schema.key?(:$ref) && + schema.dig(:items, :properties, :title).is_a?(Hash) + end + + def rewrite_candidates_property(properties) + schema = properties[:candidates] + return unless candidates_hull?(schema) + + properties[:candidates] = SELECTOR_CANDIDATES_REF.dup + end + + def candidates_hull?(schema) + return false unless schema.is_a?(Hash) && !schema.key?(:$ref) + + properties = schema[:properties] + properties.is_a?(Hash) && bucket_names.all? { properties[it].is_a?(Hash) } + end + end + end +end diff --git a/spec/support/openapi/json_schema.rb b/spec/support/openapi/json_schema.rb new file mode 100644 index 000000000..d81049a56 --- /dev/null +++ b/spec/support/openapi/json_schema.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +require 'html2rss' + +## +# Lifts +Html2rss::Config::Schema.json_schema+'s selectors subtree into OpenAPI 3.1 +# component schemas. One walk copies, allowlists keywords, rewrites +$ref+, and +# hoists the transitive +$defs+ closure. +module Openapi + module JsonSchema + ## + # Raised when a schema node uses a keyword outside the mounted allowlist. + class Unsupported < StandardError; end + + ROOT_NAME = 'SelectorsDocument' + # Keywords on the mounted selectors closure, plus +exclusiveMaximum+. + ALLOWED_KEYWORDS = %w[ + $ref $schema additionalProperties const default description enum examples + exclusiveMaximum exclusiveMinimum items maxLength minItems minLength not + oneOf pattern patternProperties properties required title type + ].to_set.freeze + MAP_APPLICATORS = %w[properties patternProperties].to_set.freeze + SCHEMA_APPLICATORS = %w[additionalProperties items not].to_set.freeze + ARRAY_APPLICATORS = %w[oneOf].to_set.freeze + # Irregular $defs path segments → OpenAPI component CamelCase (no plural chomp). + SEGMENT_CAMEL = { + 'post_processors' => 'PostProcessor', + 'extractors' => 'Extractor' + }.freeze + private_constant :MAP_APPLICATORS, :SCHEMA_APPLICATORS, :ARRAY_APPLICATORS, :SEGMENT_CAMEL + + class << self + ## + # Translates the selectors slice into OpenAPI component schemas. + # + # @param document [Hash] JSON Schema document; defaults to the gem schema + # @return [Hash{String=>Hash}] component name → OpenAPI 3.1 schema object + # @raise [Openapi::JsonSchema::Unsupported] on an unknown keyword + def components(document = Html2rss::Config::Schema.json_schema) + source = deep_stringify(document) + selectors = source.dig('properties', 'selectors') + raise Unsupported, 'missing /properties/selectors' unless selectors.is_a?(Hash) + + collected = {} + root = translate(selectors, '/properties/selectors', source.fetch('$defs', {}), collected) + { ROOT_NAME => root }.merge(collected.sort.to_h) + end + + private + + # @param node [Object] + # @param pointer [String] + # @param defs [Hash] + # @param collected [Hash{String=>Hash}] + # @return [Object] + def translate(node, pointer, defs, collected) + case node + in Hash then translate_object(node, pointer, defs, collected) + in Array + node.map.with_index { |item, i| translate(item, "#{pointer}/#{i}", defs, collected) } + else node + end + end + + # @param node [Hash] + # @param pointer [String] + # @param defs [Hash] + # @param collected [Hash{String=>Hash}] + # @return [Hash] + # rubocop:disable-next Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity -- single keyword dispatch + def translate_object(node, pointer, defs, collected) + node.each_with_object({}) do |(key, value), result| + raise Unsupported, "#{pointer}/#{key}" unless ALLOWED_KEYWORDS.include?(key) + next if key == '$schema' + + child = "#{pointer}/#{key}" + result[key] = + if key == '$ref' + rewrite_ref(value, child, defs, collected) + elsif MAP_APPLICATORS.include?(key) + raise Unsupported, child unless value.is_a?(Hash) + + value.to_h { |n, s| [n.to_s, translate(s, "#{child}/#{n}", defs, collected)] } + elsif SCHEMA_APPLICATORS.include?(key) + value.is_a?(Hash) ? translate(value, child, defs, collected) : value + elsif ARRAY_APPLICATORS.include?(key) + raise Unsupported, child unless value.is_a?(Array) + + value.map.with_index { |s, i| translate(s, "#{child}/#{i}", defs, collected) } + else + deep_stringify(value) + end + end + end + + # @param ref [Object] + # @param pointer [String] + # @param defs [Hash] + # @param collected [Hash{String=>Hash}] + # @return [String] + def rewrite_ref(ref, pointer, defs, collected) + raise Unsupported, pointer unless ref.is_a?(String) && ref.start_with?('#/$defs/') + + def_path = ref.delete_prefix('#/$defs/') + name = component_name(def_path) + unless collected.key?(name) + collected[name] = translate(resolve_def(defs, def_path, pointer), "#/$defs/#{def_path}", defs, collected) + end + "#/components/schemas/#{name}" + end + + # @param defs [Hash] + # @param def_path [String] + # @param pointer [String] + # @return [Hash] + def resolve_def(defs, def_path, pointer) + body = def_path.split('/').reduce(defs) do |node, segment| + raise Unsupported, pointer unless node.is_a?(Hash) && node.key?(segment) + + node.fetch(segment) + end + raise Unsupported, pointer unless body.is_a?(Hash) + + body + end + + # @param def_path [String] + # @return [String] + def component_name(def_path) + camel = def_path.split('/').map { camelize_segment(it) }.join + name = "Html2rss#{camel}" + raise Unsupported, "invalid component name #{name}" unless name.match?(/\A[A-Za-z0-9._-]+\z/) + + name + end + + # @param segment [String] + # @return [String] + def camelize_segment(segment) + SEGMENT_CAMEL.fetch(segment) { segment.split('_').map(&:capitalize).join } + end + + # @param value [Object] + # @return [Object] + def deep_stringify(value) + case value + in Hash then value.to_h { |key, child| [key.to_s, deep_stringify(child)] } + in Array then value.map { deep_stringify(it) } + else value + end + end + end + end +end diff --git a/spec/support/openapi/selectors_contract_fixtures.rb b/spec/support/openapi/selectors_contract_fixtures.rb new file mode 100644 index 000000000..e4fd3388a --- /dev/null +++ b/spec/support/openapi/selectors_contract_fixtures.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +## +# Shared accept/reject payloads for SelectorsDocument ↔ OpenAPI parity. +# +# Accept rows are selectors subtrees (the value create posts under +selectors+). +# Reject rows are full client fragments for {Html2rss::Web::SelectorsDocument.from_client}. +module SelectorsContractFixtures + EMPTY = {}.freeze + ITEMS_ONLY = { items: { selector: 'article' }.freeze }.freeze + ORDER_REVERSE = { items: { selector: 'article', order: 'reverse' }.freeze }.freeze + # Reused by Phase 05 create→token coverage. + PAGINATION_INTEGER = { items: { selector: 'article', pagination: 2 }.freeze }.freeze + PAGINATION_REL_NEXT = { + items: { selector: 'article', pagination: { strategy: 'rel_next' }.freeze }.freeze + }.freeze + + ACCEPT = [ + ['empty', EMPTY], + ['items-only', ITEMS_ONLY], + ['items.order reverse', ORDER_REVERSE], + ['items.pagination integer', PAGINATION_INTEGER], + ['items.pagination rel_next', PAGINATION_REL_NEXT] + ].map(&:freeze).freeze + + REJECT = [ + ['empty items.selector', { selectors: { items: { selector: '' }.freeze }.freeze }.freeze], + [ + 'denied root channel', + { channel: { url: 'https://evil.example' }.freeze, selectors: ITEMS_ONLY }.freeze + ] + ].map(&:freeze).freeze +end