From 4ded8bfc514382ee604972263a2b7c148e03c8ed Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 10:10:20 +1000
Subject: [PATCH 1/7] feat(import): deterministic sidebar discovery for Fern
 sites

---
 src/commands/import.js      | 253 +++++++++++++++++++++++++++++++++++-
 src/commands/import.test.js | 131 +++++++++++++++++++
 2 files changed, 380 insertions(+), 4 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index 58a32ae..e591ff2 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -501,11 +501,31 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   }
   console.log()
 
+  // Fern sites ship the canonical sidebar in RSC flight chunks on every
+  // page — deterministic structure where the Firecrawl scrape usually fails
+  // acceptance and llms.txt clustering invents categories.
+  let fernNav = null
+  if (!mintlifyNav) {
+    styles.info(`Probing for Fern nav tree...`)
+    const fernStart = Date.now()
+    fernNav = await timePhase('fern probe', () => tryFernNav(sourceUrl.toString(), knownUrls))
+    if (fernNav) {
+      const pageCount = fernNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+      styles.ok(
+        `Found Fern nav tree in ${styles.bold(formatDuration(Date.now() - fernStart))} — ${styles.bold(String(fernNav.categories.length))} categor${fernNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`,
+      )
+    }
+    console.log()
+  }
+  if (debugSnapshots) {
+    debugSnapshots[`02a2-fern-nav${dbgSuffix}.json`] = fernNav ? JSON.parse(JSON.stringify(fernNav)) : null
+  }
+
   // Archbee sites embed the canonical document tree in Next.js page data.
   // Their llms.txt export can be a flat, shuffled list, so prefer the tree
   // when present.
   let archbeeNav = null
-  if (!mintlifyNav) {
+  if (!mintlifyNav && !fernNav) {
     styles.info(`Probing for Archbee document tree...`)
     const archbeeStart = Date.now()
     archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey))
@@ -523,6 +543,8 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   const scrapeDiagnostics = {}
   if (mintlifyNav) {
     scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories }
+  } else if (fernNav) {
+    scraped = { title: fernNav.title, categories: fernNav.categories }
   } else if (archbeeNav) {
     scraped = { title: archbeeNav.title, categories: archbeeNav.categories }
   } else {
@@ -1454,6 +1476,229 @@ function archbeeNodeUrl(origin, urlKey) {
   return `${origin}/${key.replace(/^\/+/, '')}`
 }
 
+// Max page fetches while filling in non-active Fern tab trees (one per tab).
+const FERN_TAB_FETCH_CAP = 8
+
+const FERN_SECTION_NODE_TYPES = new Set(['section', 'apiReference', 'apiPackage'])
+const FERN_SKIP_NODE_TYPES = new Set(['link', 'changelog', 'tab'])
+
+/**
+ * Fern-powered docs sites embed the complete nav tree in Next.js RSC flight
+ * chunks (`self.__next_f.push([1,"..."])`) on every page — titles, slugs,
+ * nesting, and order exactly as the live sidebar renders them. Only the
+ * active tab's tree is populated in a given page's payload; other tabs are
+ * listed with a `pointsTo` entry path, so each one costs a single extra
+ * fetch.
+ *
+ * Always fetches directly (never Firecrawl): the flight chunks are only
+ * guaranteed present in the raw SSR HTML, not in a rendered/processed DOM.
+ *
+ * Returns { title, categories } or null if the site is not Fern or the
+ * payload yields no usable tree. Any parse failure returns null so the
+ * probe chain falls through to the existing routes.
+ */
+async function tryFernNav(sourceUrl, knownPages) {
+  try {
+    const origin = new URL(sourceUrl).origin
+    const entryHtml = await fetchHtmlDirect(toBrowsableUrl(sourceUrl))
+    if (!entryHtml) return null
+    const isFern = /<aside[^>]*id="fern-sidebar"/.test(entryHtml) || /<meta[^>]+name="generator"[^>]+content="[^"]*buildwithfern/i.test(entryHtml)
+    if (!isFern) return null
+
+    const byPath = new Map()
+    for (const p of knownPages) byPath.set(normalizePath(p.url), p)
+
+    const ctx = { origin, byPath, seenPaths: new Set() }
+    const categories = []
+    const collect = (html, tabTitle) => {
+      const blob = decodeFernFlightBlob(html)
+      addFernCategories(categories, extractFernNavNodes(blob), { ...ctx, tabTitle })
+      return blob
+    }
+
+    const entryBlob = collect(entryHtml, null)
+    const entryPath = normalizePath(sourceUrl)
+    const seenTabTargets = new Set()
+    let tabFetches = 0
+    for (const tab of extractFernTabs(entryBlob)) {
+      if (seenTabTargets.has(tab.pointsTo)) continue
+      seenTabTargets.add(tab.pointsTo)
+      const tabUrl = `${origin}/${tab.pointsTo}`
+      if (normalizePath(tabUrl) === entryPath) continue
+      if (tabFetches >= FERN_TAB_FETCH_CAP) break
+      tabFetches++
+      const tabHtml = await fetchHtmlDirect(tabUrl)
+      if (!tabHtml) continue
+      collect(tabHtml, tab.title)
+    }
+
+    const pageCount = categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+    if (pageCount === 0) return null
+    return { title: null, categories }
+  } catch {
+    return null
+  }
+}
+
+function decodeFernFlightBlob(html) {
+  const chunks = []
+  const chunkRe = /self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g
+  let m
+  while ((m = chunkRe.exec(html)) !== null) {
+    try {
+      chunks.push(JSON.parse(`"${m[1]}"`))
+    } catch { }
+  }
+  return chunks.join('')
+}
+
+function extractBalancedObject(str, start) {
+  let depth = 0
+  let inString = false
+  for (let i = start; i < str.length; i++) {
+    const ch = str[i]
+    if (inString) {
+      if (ch === '\\') i++
+      else if (ch === '"') inString = false
+    } else if (ch === '"') inString = true
+    else if (ch === '{') depth++
+    else if (ch === '}') {
+      depth--
+      if (depth === 0) return str.slice(start, i + 1)
+    }
+  }
+  return null
+}
+
+/**
+ * Pull top-level nav container/section objects out of a decoded flight blob.
+ * The blob as a whole is not JSON — nav nodes are embedded mid-stream — so
+ * each marker hit is balanced-brace extracted and parsed on its own. Matches
+ * that fall inside an already-extracted object are its children and skipped.
+ */
+function extractFernNavNodes(blob) {
+  const nodes = []
+  const markerRe = /\{"type":"(?:sidebarRoot|sidebarGroup|section|apiReference|apiPackage)"/g
+  let consumedUpTo = -1
+  let m
+  while ((m = markerRe.exec(blob)) !== null) {
+    if (m.index < consumedUpTo) continue
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    consumedUpTo = m.index + raw.length
+    try {
+      nodes.push(JSON.parse(raw))
+    } catch { }
+  }
+  return nodes
+}
+
+function extractFernTabs(blob) {
+  const tabs = []
+  const tabRe = /\{"type":"tab",/g
+  let m
+  while ((m = tabRe.exec(blob)) !== null) {
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    let tab
+    try {
+      tab = JSON.parse(raw)
+    } catch {
+      continue
+    }
+    const pointsTo = fernField(tab.pointsTo)
+    if (!pointsTo) continue
+    tabs.push({ title: fernField(tab.title), pointsTo })
+  }
+  return tabs
+}
+
+// Absent fields serialize as the literal string "$undefined" in flight payloads.
+function fernField(value) {
+  return typeof value === 'string' && value !== '$undefined' && value !== '' ? value : null
+}
+
+function fernPageFromNode(node, ctx) {
+  if (!node || typeof node !== 'object' || node.hidden === true) return null
+  if (FERN_SKIP_NODE_TYPES.has(node.type)) return null
+  const children = Array.isArray(node.children) ? node.children.map((c) => fernPageFromNode(c, ctx)).filter(Boolean) : []
+  if (FERN_SECTION_NODE_TYPES.has(node.type)) {
+    // Sections with their own overview page render it at the section slug;
+    // pointsTo is absent there (it otherwise names the first child).
+    const landing = fernField(node.pointsTo) || (fernField(node.overviewPageId) ? fernField(node.slug) : null)
+    if (children.length === 0 && !landing) return null
+    const landingUrl = landing ? `${ctx.origin}/${landing}` : null
+    const known = landingUrl ? ctx.byPath.get(normalizePath(landingUrl)) : null
+    return {
+      title: fernField(node.title) || 'Untitled',
+      url: landingUrl ? known?.url || landingUrl : null,
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  const slug = fernField(node.slug)
+  if (slug) {
+    const url = `${ctx.origin}/${slug}`
+    const known = ctx.byPath.get(normalizePath(url))
+    return {
+      title: known?.title || fernField(node.title) || slug,
+      url: known?.url || url,
+      ...(known?.description ? { description: known.description } : {}),
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  if (children.length === 1) return children[0]
+  if (children.length > 1) return { title: fernField(node.title) || 'Untitled', url: null, pages: children }
+  return null
+}
+
+/**
+ * Top-level sections become categories; loose top-level pages (tabs without
+ * sections, e.g. a two-page MCP tab) collapse into one category named after
+ * the tab. Pages already claimed by an earlier category are dropped — the
+ * payload repeats trees across fetches, and multi-product sites nest the
+ * same API groups under both a product tab and the API-reference tab.
+ */
+function addFernCategories(categories, nodes, ctx) {
+  const roots = []
+  for (const node of nodes) {
+    if (node.type === 'sidebarRoot' || node.type === 'sidebarGroup') roots.push(...(node.children || []))
+    else roots.push(node)
+  }
+  const loosePages = []
+  for (const root of roots) {
+    if (FERN_SECTION_NODE_TYPES.has(root.type)) {
+      const section = fernPageFromNode(root, ctx)
+      if (!section) continue
+      const pages = section.pages || []
+      if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
+        pages.unshift({ title: section.title, url: section.url })
+      }
+      const deduped = dedupeFernPages(pages, ctx.seenPaths)
+      if (deduped.length > 0) categories.push({ title: section.title, pages: deduped })
+    } else {
+      const page = fernPageFromNode(root, ctx)
+      if (page) loosePages.push(page)
+    }
+  }
+  const deduped = dedupeFernPages(loosePages, ctx.seenPaths)
+  if (deduped.length > 0) categories.push({ title: ctx.tabTitle || 'Documentation', pages: deduped })
+}
+
+function dedupeFernPages(pages, seenPaths) {
+  const out = []
+  for (const p of pages) {
+    const urlSeen = p.url ? seenPaths.has(normalizePath(p.url)) : false
+    if (p.url && !urlSeen) seenPaths.add(normalizePath(p.url))
+    const kids = p.pages ? dedupeFernPages(p.pages, seenPaths) : []
+    if (kids.length === 0 && (urlSeen || !p.url)) continue
+    const entry = { ...p, url: urlSeen ? null : p.url }
+    if (kids.length > 0) entry.pages = kids
+    else delete entry.pages
+    out.push(entry)
+  }
+  return out
+}
+
 /**
  * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has
  * multiple section headers (hierarchy) and tens of links; secondary navs
@@ -2471,10 +2716,10 @@ function slotOrphansByPath(scraped, knownPages) {
   const matched = new Set()
   const pathToCategory = new Map() // normalizedPath → category
   for (const cat of scraped.categories) {
-    for (const p of cat.pages) {
+    for (const p of collectUrlPagesDeep(cat.pages)) {
       const norm = normalizePath(p.url)
       matched.add(norm)
-      pathToCategory.set(norm, cat)
+      if (!pathToCategory.has(norm)) pathToCategory.set(norm, cat)
     }
   }
 
@@ -4737,7 +4982,7 @@ function makeIconPicker() {
   }
 }
 
-export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl, resolveDocsBaseUrl, inCandidateScope }
+export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl, resolveDocsBaseUrl, inCandidateScope, tryFernNav }
 
 function formatDuration(ms) {
   const safe = Math.max(0, ms)
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index 0824f78..cc158b1 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -190,3 +190,134 @@ test('inCandidateScope normalizes trailing slashes and enforces segment boundari
   assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/docsomething'), false)
   assert.equal(__test__.inCandidateScope(candidate, 'https://example.com/'), false)
 })
+
+function fernHtml(fragments, { aside = true, generator = false, extraBody = '' } = {}) {
+  const scripts = fragments.map((fragment) => `<script>self.__next_f.push([1,${JSON.stringify(fragment)}])</script>`).join('\n')
+  const head = generator ? '<meta name="generator" content="https://buildwithfern.com">' : ''
+  const sidebar = aside ? '<aside id="fern-sidebar" class="fern-sidebar-container"></aside>' : ''
+  return `<!doctype html><html><head>${head}</head><body>${sidebar}${extraBody}${scripts}</body></html>`
+}
+
+function mockHtmlFetch(pages) {
+  globalThis.fetch = async (url) => {
+    const entry = pages[String(url)]
+    if (entry === undefined) return { ok: false, status: 404, url: String(url), text: async () => '' }
+    if (entry instanceof Error) throw entry
+    return { ok: true, status: 200, url: String(url), text: async () => entry }
+  }
+}
+
+const GEN1_TREE =
+  '7a:["$","$L7b",null,{"children":[' +
+  '{"type":"sidebarGroup","id":"sidebar-group:section:docs/get-started","collapsed":"$undefined","children":[' +
+  '{"type":"section","id":"section:docs/get-started","title":"Get started","slug":"docs/get-started","hidden":false,"pointsTo":"$undefined","children":[' +
+  '{"type":"page","id":"page:intro","title":"Introduction","slug":"intro","hidden":false},' +
+  '{"type":"page","id":"page:secret","title":"Secret","slug":"secret","hidden":true},' +
+  '{"type":"section","id":"section:docs/nested","title":"Nested","slug":"nested/overview","hidden":false,"pointsTo":"$undefined","overviewPageId":"nested/overview.mdx","children":[' +
+  '{"type":"page","id":"page:nested/deep","title":"Deep page","slug":"nested/deep","hidden":false}]}]}]}]}]'
+
+const GEN2_TREE =
+  '7a:["$","$L7b",null,{"children":[' +
+  '{"type":"sidebarGroup","id":"c00ed6ec6366da0ae76c9bd27e2e2c23bf3785a61bbb0990","collapsed":"$undefined","children":[' +
+  '{"type":"section","id":"28eaf69e90e0a08edb3d6840439eeacb0b9dbdb3","title":"Get started","slug":"docs/get-started","hidden":false,"pointsTo":"$undefined","children":[' +
+  '{"type":"page","id":"bc59f5966f28f9f0c31ac71f02619db1e70e8c65","title":"Introduction","slug":"intro","hidden":false},' +
+  '{"type":"page","id":"3d9a0da0ad64e4f360afee1dee83e4ae1b848dff","title":"Secret","slug":"secret","hidden":true},' +
+  '{"type":"section","id":"75861c60edfc32eefed7d2a77f7d1b9563a7bb74","title":"Nested","slug":"docs/nested","hidden":false,"pointsTo":"nested/overview","children":[' +
+  '{"type":"page","id":"03b7470af4ce00f9a9dc979ba32fec3fd1c4a7bd","title":"Deep page","slug":"nested/deep","hidden":false}]}]}]}]}]'
+
+function assertGetStartedTree(nav) {
+  assert.equal(nav.categories.length, 1)
+  const [category] = nav.categories
+  assert.equal(category.title, 'Get started')
+  assert.deepEqual(
+    category.pages.map((p) => p.url),
+    ['https://fern.example/intro', 'https://fern.example/nested/overview'],
+  )
+  assert.equal(category.pages[0].title, 'Introduction')
+  assert(!category.pages.some((p) => p.title === 'Secret'))
+  const nested = category.pages[1]
+  assert.equal(nested.title, 'Nested')
+  assert.deepEqual(nested.pages, [{ title: 'Deep page', url: 'https://fern.example/nested/deep' }])
+}
+
+test('tryFernNav parses the page:<slug> id generation into nested categories, skipping hidden pages', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE]) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assertGetStartedTree(nav)
+})
+
+test('tryFernNav parses the hashed id generation into the same categories', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN2_TREE]) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assertGetStartedTree(nav)
+})
+
+test('tryFernNav detects via generator meta when the sidebar aside is absent', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE], { aside: false, generator: true }) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assertGetStartedTree(nav)
+})
+
+test('tryFernNav returns null when neither detection signal is present', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE], { aside: false }) })
+  assert.equal(await __test__.tryFernNav('https://fern.example/intro', []), null)
+})
+
+test('tryFernNav ignores prose mentions of the detection signals', async () => {
+  mockHtmlFetch({
+    'https://fern.example/intro': fernHtml([GEN1_TREE], {
+      aside: false,
+      extraBody: '<p>Docs generated with buildwithfern.com use an aside with id="fern-sidebar".</p>',
+    }),
+  })
+  assert.equal(await __test__.tryFernNav('https://fern.example/intro', []), null)
+})
+
+test('tryFernNav returns null when the page has no flight chunks', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([]) })
+  assert.equal(await __test__.tryFernNav('https://fern.example/intro', []), null)
+})
+
+test('tryFernNav prefers llms.txt titles, urls, and descriptions over payload fields', async () => {
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE]) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [
+    { title: 'Welcome', url: 'https://fern.example/intro.md', description: 'Start here' },
+  ])
+  assert.deepEqual(nav.categories[0].pages[0], { title: 'Welcome', url: 'https://fern.example/intro.md', description: 'Start here' })
+})
+
+const TAB_LIST =
+  '5f:["$","$L60",null,{"tabs":[' +
+  '{"type":"tab","id":"tab:docs","title":"Docs","slug":"docs","hidden":false,"pointsTo":"intro","child":{"type":"sidebarRoot","id":"sidebar:|||aa11","collapsed":"$undefined","children":[]}},' +
+  '{"type":"tab","id":"tab:api","title":"API Reference","slug":"api","hidden":false,"pointsTo":"api/start","child":{"type":"sidebarRoot","id":"sidebar:|||bb22","collapsed":"$undefined","children":[]}}]}]'
+
+const API_TAB_TREE =
+  '9c:["$","$L9d",null,{"children":[' +
+  '{"type":"apiPackage","id":"api-pkg:api/pets","title":"Pets","slug":"api/pets","hidden":false,"pointsTo":"api/start","children":[' +
+  '{"type":"endpoint","id":"api-leaf:api/start","title":"List pets","slug":"api/start","hidden":false,"method":"GET"},' +
+  '{"type":"endpoint","id":"api-leaf:api/pets/create","title":"Create pet","slug":"api/pets/create","hidden":false,"method":"POST"}]}]}]'
+
+test('tryFernNav fetches non-active tabs once and merges their trees', async () => {
+  mockHtmlFetch({
+    'https://fern.example/intro': fernHtml([GEN1_TREE, TAB_LIST]),
+    'https://fern.example/api/start': fernHtml([API_TAB_TREE]),
+  })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assert.deepEqual(
+    nav.categories.map((c) => c.title),
+    ['Get started', 'Pets'],
+  )
+  assert.deepEqual(
+    nav.categories[1].pages.map((p) => p.url),
+    ['https://fern.example/api/start', 'https://fern.example/api/pets/create'],
+  )
+})
+
+test('tryFernNav keeps the entry tree when a tab fetch fails', async () => {
+  mockHtmlFetch({
+    'https://fern.example/intro': fernHtml([GEN1_TREE, TAB_LIST]),
+    'https://fern.example/api/start': new Error('network down'),
+  })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assertGetStartedTree(nav)
+})

From ecc0888787f13ab1da9aa9631ba69fe8c0ed9f20 Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 13:38:07 +1000
Subject: [PATCH 2/7] fix(import): keep Fern-authored categories flat instead
 of re-nesting by URL

---
 src/commands/import.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/commands/import.js b/src/commands/import.js
index e591ff2..259c0d6 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -823,7 +823,14 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   // nestByUrlHierarchy renders it as the real parent of its children.
   await injectSectionLandingPages(organized, sourceUrl)
 
+  // A Fern tree is the authored sidebar: re-nesting by URL invents parent
+  // folders the source site renders flat (a section landing absorbing its
+  // path-descendant siblings, e.g. vapi's /server-url under Webhooks). Only
+  // the sweep-synthesized categories still nest — their pages were moved in
+  // flat and have no authored structure.
+  const sweepCategories = new Set(['API Reference', 'Changelog'])
   for (const cat of organized.categories || []) {
+    if (fernNav && !sweepCategories.has(cat.title)) continue
     cat.pages = nestByUrlHierarchy(cat.pages)
   }
 

From 62ce9b5aed0bf34e0f6b77db4f4af6c364dfbab7 Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 13:57:25 +1000
Subject: [PATCH 3/7] fix(import): fern pointsTo is a first-child alias, not a
 section page

---
 src/commands/import.js      | 21 +++++++++++++--------
 src/commands/import.test.js | 26 +++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index 259c0d6..3aa3e26 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -1630,15 +1630,20 @@ function fernPageFromNode(node, ctx) {
   if (FERN_SKIP_NODE_TYPES.has(node.type)) return null
   const children = Array.isArray(node.children) ? node.children.map((c) => fernPageFromNode(c, ctx)).filter(Boolean) : []
   if (FERN_SECTION_NODE_TYPES.has(node.type)) {
-    // Sections with their own overview page render it at the section slug;
-    // pointsTo is absent there (it otherwise names the first child).
-    const landing = fernField(node.pointsTo) || (fernField(node.overviewPageId) ? fernField(node.slug) : null)
-    if (children.length === 0 && !landing) return null
-    const landingUrl = landing ? `${ctx.origin}/${landing}` : null
-    const known = landingUrl ? ctx.byPath.get(normalizePath(landingUrl)) : null
+    // A section owns a page only when overviewPageId is set — it then renders
+    // at the section's own slug. pointsTo is a navigation alias to the
+    // section's first descendant page, never a page of the section itself.
+    const landing = fernField(node.overviewPageId) ? fernField(node.slug) : null
+    const title = fernField(node.title) || 'Untitled'
+    if (!landing) {
+      if (children.length === 0) return null
+      return { title, url: null, _emptyParent: true, _virtualPathSegs: [kebabCase(title) || 'group'], pages: children }
+    }
+    const landingUrl = `${ctx.origin}/${landing}`
+    const known = ctx.byPath.get(normalizePath(landingUrl))
     return {
-      title: fernField(node.title) || 'Untitled',
-      url: landingUrl ? known?.url || landingUrl : null,
+      title,
+      url: known?.url || landingUrl,
       ...(children.length > 0 ? { pages: children } : {}),
     }
   }
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index cc158b1..7ae9f81 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -222,7 +222,7 @@ const GEN2_TREE =
   '{"type":"section","id":"28eaf69e90e0a08edb3d6840439eeacb0b9dbdb3","title":"Get started","slug":"docs/get-started","hidden":false,"pointsTo":"$undefined","children":[' +
   '{"type":"page","id":"bc59f5966f28f9f0c31ac71f02619db1e70e8c65","title":"Introduction","slug":"intro","hidden":false},' +
   '{"type":"page","id":"3d9a0da0ad64e4f360afee1dee83e4ae1b848dff","title":"Secret","slug":"secret","hidden":true},' +
-  '{"type":"section","id":"75861c60edfc32eefed7d2a77f7d1b9563a7bb74","title":"Nested","slug":"docs/nested","hidden":false,"pointsTo":"nested/overview","children":[' +
+  '{"type":"section","id":"75861c60edfc32eefed7d2a77f7d1b9563a7bb74","title":"Nested","slug":"nested/overview","hidden":false,"pointsTo":"$undefined","overviewPageId":"nested/overview.mdx","children":[' +
   '{"type":"page","id":"03b7470af4ce00f9a9dc979ba32fec3fd1c4a7bd","title":"Deep page","slug":"nested/deep","hidden":false}]}]}]}]}]'
 
 function assertGetStartedTree(nav) {
@@ -286,6 +286,30 @@ test('tryFernNav prefers llms.txt titles, urls, and descriptions over payload fi
   assert.deepEqual(nav.categories[0].pages[0], { title: 'Welcome', url: 'https://fern.example/intro.md', description: 'Start here' })
 })
 
+test('tryFernNav treats section pointsTo as an alias: the first child keeps its page and the section becomes a titled group', async () => {
+  const tree =
+    '7a:["$","$L7b",null,{"children":[' +
+    '{"type":"sidebarGroup","id":"sidebar-group:section:docs/guides","children":[' +
+    '{"type":"section","id":"section:docs/guides","title":"Guides","slug":"docs/guides","hidden":false,"pointsTo":"$undefined","overviewPageId":"$undefined","children":[' +
+    '{"type":"section","id":"section:docs/guides/billing","title":"Billing","slug":"docs/guides/billing","hidden":false,"pointsTo":"billing/manage","overviewPageId":"$undefined","children":[' +
+    '{"type":"page","id":"page:billing/manage","title":"Manage billing","slug":"billing/manage","hidden":false},' +
+    '{"type":"page","id":"page:billing/limits","title":"Billing limits","slug":"billing/limits","hidden":false}]}]}]}]}]'
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([tree]) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  const [category] = nav.categories
+  assert.equal(category.title, 'Guides')
+  const billing = category.pages[0]
+  assert.equal(billing.title, 'Billing')
+  assert.equal(billing.url, null)
+  assert.deepEqual(
+    billing.pages.map((p) => [p.title, p.url]),
+    [
+      ['Manage billing', 'https://fern.example/billing/manage'],
+      ['Billing limits', 'https://fern.example/billing/limits'],
+    ],
+  )
+})
+
 const TAB_LIST =
   '5f:["$","$L60",null,{"tabs":[' +
   '{"type":"tab","id":"tab:docs","title":"Docs","slug":"docs","hidden":false,"pointsTo":"intro","child":{"type":"sidebarRoot","id":"sidebar:|||aa11","collapsed":"$undefined","children":[]}},' +

From 7e197666c475e51266d41f76fbe805e4982d2edc Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 14:58:58 +1000
Subject: [PATCH 4/7] fix(import): sidebar labels win over llms.txt heading
 titles on Fern sites

---
 src/commands/import.js      | 4 +++-
 src/commands/import.test.js | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index 3aa3e26..124ebe4 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -1652,7 +1652,9 @@ function fernPageFromNode(node, ctx) {
     const url = `${ctx.origin}/${slug}`
     const known = ctx.byPath.get(normalizePath(url))
     return {
-      title: known?.title || fernField(node.title) || slug,
+      // The payload title is the sidebar label as rendered; llms.txt titles
+      // come from page headings, which can differ from the nav.
+      title: fernField(node.title) || known?.title || slug,
       url: known?.url || url,
       ...(known?.description ? { description: known.description } : {}),
       ...(children.length > 0 ? { pages: children } : {}),
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index 7ae9f81..459e729 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -278,12 +278,12 @@ test('tryFernNav returns null when the page has no flight chunks', async () => {
   assert.equal(await __test__.tryFernNav('https://fern.example/intro', []), null)
 })
 
-test('tryFernNav prefers llms.txt titles, urls, and descriptions over payload fields', async () => {
+test('tryFernNav keeps the sidebar title but takes urls and descriptions from llms.txt', async () => {
   mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE]) })
   const nav = await __test__.tryFernNav('https://fern.example/intro', [
     { title: 'Welcome', url: 'https://fern.example/intro.md', description: 'Start here' },
   ])
-  assert.deepEqual(nav.categories[0].pages[0], { title: 'Welcome', url: 'https://fern.example/intro.md', description: 'Start here' })
+  assert.deepEqual(nav.categories[0].pages[0], { title: 'Introduction', url: 'https://fern.example/intro.md', description: 'Start here' })
 })
 
 test('tryFernNav treats section pointsTo as an alias: the first child keeps its page and the section becomes a titled group', async () => {

From df7189b2c17279dd3edb0d5bcf719230c3b070d0 Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 19:26:28 +1000
Subject: [PATCH 5/7] fix(import): fern section overview becomes the category's
 parent page

---
 src/commands/import.js      |  9 +++++----
 src/commands/import.test.js | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index 124ebe4..f80fd11 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -1683,10 +1683,11 @@ function addFernCategories(categories, nodes, ctx) {
     if (FERN_SECTION_NODE_TYPES.has(root.type)) {
       const section = fernPageFromNode(root, ctx)
       if (!section) continue
-      const pages = section.pages || []
-      if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
-        pages.unshift({ title: section.title, url: section.url })
-      }
+      // A section owning an overview page renders as a clickable parent on
+      // the source site — keep the overview as the category's parent page
+      // with the section's children nested under it. Page-less sections
+      // spread their children directly under the category header.
+      const pages = section.url ? [section] : section.pages || []
       const deduped = dedupeFernPages(pages, ctx.seenPaths)
       if (deduped.length > 0) categories.push({ title: section.title, pages: deduped })
     } else {
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index 459e729..86c7172 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -286,6 +286,28 @@ test('tryFernNav keeps the sidebar title but takes urls and descriptions from ll
   assert.deepEqual(nav.categories[0].pages[0], { title: 'Introduction', url: 'https://fern.example/intro.md', description: 'Start here' })
 })
 
+test('tryFernNav renders a top-level section with an overview page as the category parent page', async () => {
+  const tree =
+    '7a:["$","$L7b",null,{"children":[' +
+    '{"type":"sidebarGroup","id":"sidebar-group:section:webhooks","children":[' +
+    '{"type":"section","id":"section:webhooks","title":"Webhooks","slug":"server-url","hidden":false,"pointsTo":"$undefined","overviewPageId":"server-url.mdx","children":[' +
+    '{"type":"page","id":"page:server-url/events","title":"Server events","slug":"server-url/events","hidden":false}]}]}]}]'
+  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([tree]) })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assert.deepEqual(nav.categories, [
+    {
+      title: 'Webhooks',
+      pages: [
+        {
+          title: 'Webhooks',
+          url: 'https://fern.example/server-url',
+          pages: [{ title: 'Server events', url: 'https://fern.example/server-url/events' }],
+        },
+      ],
+    },
+  ])
+})
+
 test('tryFernNav treats section pointsTo as an alias: the first child keeps its page and the section becomes a titled group', async () => {
   const tree =
     '7a:["$","$L7b",null,{"children":[' +

From 72f9fb3d82d110609b7a5c56d277d545472b198b Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 20:34:40 +1000
Subject: [PATCH 6/7] Revert "fix(import): fern section overview becomes the
 category's parent page"

This reverts commit df7189b2c17279dd3edb0d5bcf719230c3b070d0.
---
 src/commands/import.js      |  9 ++++-----
 src/commands/import.test.js | 22 ----------------------
 2 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index f80fd11..124ebe4 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -1683,11 +1683,10 @@ function addFernCategories(categories, nodes, ctx) {
     if (FERN_SECTION_NODE_TYPES.has(root.type)) {
       const section = fernPageFromNode(root, ctx)
       if (!section) continue
-      // A section owning an overview page renders as a clickable parent on
-      // the source site — keep the overview as the category's parent page
-      // with the section's children nested under it. Page-less sections
-      // spread their children directly under the category header.
-      const pages = section.url ? [section] : section.pages || []
+      const pages = section.pages || []
+      if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
+        pages.unshift({ title: section.title, url: section.url })
+      }
       const deduped = dedupeFernPages(pages, ctx.seenPaths)
       if (deduped.length > 0) categories.push({ title: section.title, pages: deduped })
     } else {
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index 86c7172..459e729 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -286,28 +286,6 @@ test('tryFernNav keeps the sidebar title but takes urls and descriptions from ll
   assert.deepEqual(nav.categories[0].pages[0], { title: 'Introduction', url: 'https://fern.example/intro.md', description: 'Start here' })
 })
 
-test('tryFernNav renders a top-level section with an overview page as the category parent page', async () => {
-  const tree =
-    '7a:["$","$L7b",null,{"children":[' +
-    '{"type":"sidebarGroup","id":"sidebar-group:section:webhooks","children":[' +
-    '{"type":"section","id":"section:webhooks","title":"Webhooks","slug":"server-url","hidden":false,"pointsTo":"$undefined","overviewPageId":"server-url.mdx","children":[' +
-    '{"type":"page","id":"page:server-url/events","title":"Server events","slug":"server-url/events","hidden":false}]}]}]}]'
-  mockHtmlFetch({ 'https://fern.example/intro': fernHtml([tree]) })
-  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
-  assert.deepEqual(nav.categories, [
-    {
-      title: 'Webhooks',
-      pages: [
-        {
-          title: 'Webhooks',
-          url: 'https://fern.example/server-url',
-          pages: [{ title: 'Server events', url: 'https://fern.example/server-url/events' }],
-        },
-      ],
-    },
-  ])
-})
-
 test('tryFernNav treats section pointsTo as an alias: the first child keeps its page and the section becomes a titled group', async () => {
   const tree =
     '7a:["$","$L7b",null,{"children":[' +

From da7fc8a8413eefb005b1c4d20dd70a6d3f1c914f Mon Sep 17 00:00:00 2001
From: minhthanhdang <mtddev2004@gmail.com>
Date: Thu, 13 Aug 2026 23:39:18 +1000
Subject: [PATCH 7/7] Harden Fern nav probe: stale-entry retry, api-ref-aware
 coverage gate, tab-title fallback for untitled roots

---
 src/commands/import.js      | 51 +++++++++++++++++++++++++++++--------
 src/commands/import.test.js | 35 +++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/src/commands/import.js b/src/commands/import.js
index 124ebe4..8faf3ca 100644
--- a/src/commands/import.js
+++ b/src/commands/import.js
@@ -565,13 +565,18 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   // node because that's the one /docs page the scrape saw). Discard the
   // scrape and fall through to the llms.txt path, which uses URL-based
   // clustering when multiple files were merged.
+  // API-reference pages are excluded from the denominator: sidebars routinely
+  // omit generated endpoint stubs, and those pages are swept into reference/
+  // regardless of nav quality, so they say nothing about the nav's fitness as
+  // the import's spine.
   let scrapeDiscardedForCoverage = false
   if (scraped && llms && knownUrls.length > 0) {
     const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
-    const coverage = scrapedPages / knownUrls.length
+    const nonReferenceKnown = knownUrls.filter((p) => !urlIsApiReference(p.url))
+    const coverage = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1
     if (coverage < 0.75) {
       styles.info(
-        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%) — discarding scrape and organizing from llms.txt.`,
+        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%${nonReferenceKnown.length < knownUrls.length ? `, ${knownUrls.length - nonReferenceKnown.length} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`,
       )
       scraped = null
       scrapeDiscardedForCoverage = true
@@ -1499,6 +1504,8 @@ const FERN_SKIP_NODE_TYPES = new Set(['link', 'changelog', 'tab'])
  *
  * Always fetches directly (never Firecrawl): the flight chunks are only
  * guaranteed present in the raw SSR HTML, not in a rendered/processed DOM.
+ * A dead entry URL (moved slug) is retried once from a known page, since
+ * Fern soft-404s stale deep links.
  *
  * Returns { title, categories } or null if the site is not Fern or the
  * payload yields no usable tree. Any parse failure returns null so the
@@ -1507,10 +1514,18 @@ const FERN_SKIP_NODE_TYPES = new Set(['link', 'changelog', 'tab'])
 async function tryFernNav(sourceUrl, knownPages) {
   try {
     const origin = new URL(sourceUrl).origin
-    const entryHtml = await fetchHtmlDirect(toBrowsableUrl(sourceUrl))
-    if (!entryHtml) return null
-    const isFern = /<aside[^>]*id="fern-sidebar"/.test(entryHtml) || /<meta[^>]+name="generator"[^>]+content="[^"]*buildwithfern/i.test(entryHtml)
-    if (!isFern) return null
+    const looksFern = (html) => /<aside[^>]*id="fern-sidebar"/.test(html) || /<meta[^>]+name="generator"[^>]+content="[^"]*buildwithfern/i.test(html)
+    let entryUrl = toBrowsableUrl(sourceUrl)
+    let entryHtml = await fetchHtmlDirect(entryUrl)
+    if (!looksFern(entryHtml)) {
+      // A stale entry deep link 404s on Fern (the not-found boundary renders
+      // with a truncated tree), so retry once from a known live page.
+      const retry = knownPages.find((p) => normalizePath(p.url) !== normalizePath(sourceUrl))
+      if (!retry) return null
+      entryUrl = toBrowsableUrl(retry.url)
+      entryHtml = await fetchHtmlDirect(entryUrl)
+      if (!looksFern(entryHtml)) return null
+    }
 
     const byPath = new Map()
     for (const p of knownPages) byPath.set(normalizePath(p.url), p)
@@ -1524,7 +1539,7 @@ async function tryFernNav(sourceUrl, knownPages) {
     }
 
     const entryBlob = collect(entryHtml, null)
-    const entryPath = normalizePath(sourceUrl)
+    const entryPath = normalizePath(entryUrl)
     const seenTabTargets = new Set()
     let tabFetches = 0
     for (const tab of extractFernTabs(entryBlob)) {
@@ -1666,9 +1681,11 @@ function fernPageFromNode(node, ctx) {
 }
 
 /**
- * Top-level sections become categories; loose top-level pages (tabs without
- * sections, e.g. a two-page MCP tab) collapse into one category named after
- * the tab. Pages already claimed by an earlier category are dropped — the
+ * Top-level sections become categories; untitled ones adopt the tab title so
+ * two tabs' unnamed roots don't merge into one "Untitled" category with
+ * colliding group slugs. Loose top-level pages (tabs without sections, e.g. a
+ * two-page MCP tab) collapse into one category named after the tab. Pages
+ * already claimed by an earlier category are dropped — the
  * payload repeats trees across fetches, and multi-product sites nest the
  * same API groups under both a product tab and the API-reference tab.
  */
@@ -1683,6 +1700,7 @@ function addFernCategories(categories, nodes, ctx) {
     if (FERN_SECTION_NODE_TYPES.has(root.type)) {
       const section = fernPageFromNode(root, ctx)
       if (!section) continue
+      if (!fernField(root.title) && ctx.tabTitle) section.title = ctx.tabTitle
       const pages = section.pages || []
       if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
         pages.unshift({ title: section.title, url: section.url })
@@ -2437,9 +2455,20 @@ function reclassifyPagesByUrlSegment(scraped, { segmentRe, categoryRe, defaultTi
  * live under a separate API Reference section — this lands them all in
  * `reference/` after staging. Returns the number of pages relocated.
  */
+const API_REFERENCE_URL_SEGMENT_RE = /^(api[-_]?reference|endpoints?)$/i
+
+function urlIsApiReference(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean)
+    return segs.some((s) => API_REFERENCE_URL_SEGMENT_RE.test(s))
+  } catch {
+    return false
+  }
+}
+
 function reclassifyReferencePages(scraped) {
   return reclassifyPagesByUrlSegment(scraped, {
-    segmentRe: /^(api[-_]?reference|endpoints?)$/i,
+    segmentRe: API_REFERENCE_URL_SEGMENT_RE,
     categoryRe: /^(api[ -]?reference|reference|api|endpoints?)$/i,
     defaultTitle: 'API Reference',
   })
diff --git a/src/commands/import.test.js b/src/commands/import.test.js
index 459e729..fc2aaa2 100644
--- a/src/commands/import.test.js
+++ b/src/commands/import.test.js
@@ -278,6 +278,24 @@ test('tryFernNav returns null when the page has no flight chunks', async () => {
   assert.equal(await __test__.tryFernNav('https://fern.example/intro', []), null)
 })
 
+test('tryFernNav retries from a known page when the entry URL is dead', async () => {
+  mockHtmlFetch({ 'https://fern.example/nested/deep': fernHtml([GEN1_TREE]) })
+  const nav = await __test__.tryFernNav('https://fern.example/dead-slug', [
+    { title: 'Deep page', url: 'https://fern.example/nested/deep.md', description: undefined },
+  ])
+  assert.equal(nav.categories.length, 1)
+  assert.equal(nav.categories[0].title, 'Get started')
+  assert.deepEqual(nav.categories[0].pages[1].pages, [{ title: 'Deep page', url: 'https://fern.example/nested/deep.md' }])
+})
+
+test('tryFernNav returns null when the entry URL is dead and the known page is not Fern', async () => {
+  mockHtmlFetch({ 'https://fern.example/nested/deep': fernHtml([GEN1_TREE], { aside: false }) })
+  assert.equal(
+    await __test__.tryFernNav('https://fern.example/dead-slug', [{ title: 'Deep page', url: 'https://fern.example/nested/deep.md', description: undefined }]),
+    null,
+  )
+})
+
 test('tryFernNav keeps the sidebar title but takes urls and descriptions from llms.txt', async () => {
   mockHtmlFetch({ 'https://fern.example/intro': fernHtml([GEN1_TREE]) })
   const nav = await __test__.tryFernNav('https://fern.example/intro', [
@@ -337,6 +355,23 @@ test('tryFernNav fetches non-active tabs once and merges their trees', async ()
   )
 })
 
+test('tryFernNav names an untitled tab-tree root after its tab', async () => {
+  const untitledTabTree =
+    '9c:["$","$L9d",null,{"children":[' +
+    '{"type":"apiPackage","id":"api-pkg:api","title":"$undefined","slug":"api","hidden":false,"pointsTo":"api/start","children":[' +
+    '{"type":"endpoint","id":"api-leaf:api/start","title":"List pets","slug":"api/start","hidden":false,"method":"GET"},' +
+    '{"type":"endpoint","id":"api-leaf:api/pets/create","title":"Create pet","slug":"api/pets/create","hidden":false,"method":"POST"}]}]}]'
+  mockHtmlFetch({
+    'https://fern.example/intro': fernHtml([GEN1_TREE, TAB_LIST]),
+    'https://fern.example/api/start': fernHtml([untitledTabTree]),
+  })
+  const nav = await __test__.tryFernNav('https://fern.example/intro', [])
+  assert.deepEqual(
+    nav.categories.map((c) => c.title),
+    ['Get started', 'API Reference'],
+  )
+})
+
 test('tryFernNav keeps the entry tree when a tab fetch fails', async () => {
   mockHtmlFetch({
     'https://fern.example/intro': fernHtml([GEN1_TREE, TAB_LIST]),