diff --git a/app/note/[id].tsx b/app/note/[id].tsx index e3cb139..b1f052c 100644 --- a/app/note/[id].tsx +++ b/app/note/[id].tsx @@ -60,6 +60,7 @@ import { type PickedImageResult, } from '../../src/services/imageInsertStorage'; import { exportNotebookAsPdf } from '../../src/services/exportService'; +import { resolvePdfRenderUri } from '../../src/services/pdfUtilsModule'; import { recordSuccessfulNoteSave } from '../../src/services/lifecycleService'; import { textBoxId, insertedElementId } from '../../src/utils/id'; import { spacing } from '../../src/theme/spacing'; @@ -147,6 +148,7 @@ export default function NoteScreen() { const lastPenToolRef = useRef<'pen' | 'highlighter' | 'crayon' | 'calligraphy'>('pen'); const [metadata, setMetadata] = useState(null); + const [pdfRenderUri, setPdfRenderUri] = useState(); const [enginePages, setEnginePages] = useState([]); const [overlayMap, setOverlayMap] = useState>( () => new Map(), @@ -309,7 +311,11 @@ export default function NoteScreen() { try { const [meta, body] = await Promise.all([getNote(id), readNoteBody(id)]); if (cancelled) return; - if (meta) setMetadata(meta); + const sourceUri = meta?.pdfUri ?? undefined; + const renderUri = await resolvePdfRenderUri(sourceUri).catch(() => sourceUri); + if (cancelled) return; + setPdfRenderUri(renderUri); + setMetadata(meta); if (body.kind === 'unreadable') { // A body file exists but cannot be read. Never fall back to an empty // notebook here: autosave would overwrite the file and destroy @@ -905,7 +911,7 @@ export default function NoteScreen() { pageHeight={PAGE_HEIGHT} toolState={toolState} backgroundType={metadata?.backgroundType ?? 'plain'} - pdfBackgroundBaseUri={metadata?.pdfUri ?? undefined} + pdfBackgroundBaseUri={pdfRenderUri} fingerDrawingEnabled={fingerDrawingEnabled} onReady={handleCanvasReady} onDrawingChange={handleDrawingChange} diff --git a/ios/OpenNotes.xcodeproj/project.pbxproj b/ios/OpenNotes.xcodeproj/project.pbxproj index 7e7040d..1bc8c69 100644 --- a/ios/OpenNotes.xcodeproj/project.pbxproj +++ b/ios/OpenNotes.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 7D3A801A2E1A4CC89B1B09C1 /* ICloudBackupModule.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7D3A80182E1A4CC89B1B09C1 /* ICloudBackupModule.swift */; }; 7D3A801B2E1A4CC89B1B09C1 /* ICloudBackupModule.m in Sources */ = {isa = PBXBuildFile; fileRef = 7D3A80192E1A4CC89B1B09C1 /* ICloudBackupModule.m */; }; 7D3A801D2E1A4CC89B1B09C1 /* ImageThumbnailEncoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7D3A801C2E1A4CC89B1B09C1 /* ImageThumbnailEncoder.swift */; }; + 7D3A80322E1A4CC89B1B09C1 /* PDFRenderCopy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7D3A80312E1A4CC89B1B09C1 /* PDFRenderCopy.swift */; }; 9C2FD7C620026BB779139CCD /* libPods-OpenNotes.a in Frameworks */ = {isa = PBXBuildFile; fileRef = A7A85B485F4321855B927960 /* libPods-OpenNotes.a */; }; BB2F792D24A3F905000567C9 /* Expo.plist in Resources */ = {isa = PBXBuildFile; fileRef = BB2F792C24A3F905000567C9 /* Expo.plist */; }; F11748422D0307B40044C1D9 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F11748412D0307B40044C1D9 /* AppDelegate.swift */; }; @@ -31,6 +32,7 @@ 7D3A80182E1A4CC89B1B09C1 /* ICloudBackupModule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = ICloudBackupModule.swift; path = OpenNotes/ICloudBackupModule.swift; sourceTree = ""; }; 7D3A80192E1A4CC89B1B09C1 /* ICloudBackupModule.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; name = ICloudBackupModule.m; path = OpenNotes/ICloudBackupModule.m; sourceTree = ""; }; 7D3A801C2E1A4CC89B1B09C1 /* ImageThumbnailEncoder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = ImageThumbnailEncoder.swift; path = OpenNotes/ImageThumbnailEncoder.swift; sourceTree = ""; }; + 7D3A80312E1A4CC89B1B09C1 /* PDFRenderCopy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = PDFRenderCopy.swift; path = OpenNotes/PDFRenderCopy.swift; sourceTree = ""; }; 9B82DFB67047998EABD93C92 /* ExpoModulesProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = ExpoModulesProvider.swift; path = "Pods/Target Support Files/Pods-OpenNotes/ExpoModulesProvider.swift"; sourceTree = ""; }; A7A85B485F4321855B927960 /* libPods-OpenNotes.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-OpenNotes.a"; sourceTree = BUILT_PRODUCTS_DIR; }; AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = SplashScreen.storyboard; path = OpenNotes/SplashScreen.storyboard; sourceTree = ""; }; @@ -63,6 +65,7 @@ 7D3A80182E1A4CC89B1B09C1 /* ICloudBackupModule.swift */, 7D3A80192E1A4CC89B1B09C1 /* ICloudBackupModule.m */, 7D3A801C2E1A4CC89B1B09C1 /* ImageThumbnailEncoder.swift */, + 7D3A80312E1A4CC89B1B09C1 /* PDFRenderCopy.swift */, F11748442D0722820044C1D9 /* OpenNotes-Bridging-Header.h */, BB2F792B24A3F905000567C9 /* Supporting */, 13B07FB51A68108700A75B9A /* Images.xcassets */, @@ -352,6 +355,7 @@ 7D3A801A2E1A4CC89B1B09C1 /* ICloudBackupModule.swift in Sources */, 7D3A801B2E1A4CC89B1B09C1 /* ICloudBackupModule.m in Sources */, 7D3A801D2E1A4CC89B1B09C1 /* ImageThumbnailEncoder.swift in Sources */, + 7D3A80322E1A4CC89B1B09C1 /* PDFRenderCopy.swift in Sources */, 4788858903E15B3CAEB129E6 /* ExpoModulesProvider.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/ios/OpenNotes/PDFRenderCopy.swift b/ios/OpenNotes/PDFRenderCopy.swift new file mode 100644 index 0000000..9f8d0de --- /dev/null +++ b/ios/OpenNotes/PDFRenderCopy.swift @@ -0,0 +1,49 @@ +import CryptoKit +import Foundation +import PDFKit + +enum PDFRenderCopy { + private static let unannotated: NSCache = { + let cache = NSCache() + cache.countLimit = 128 + return cache + }() + + static func url(for source: URL, cacheDirectory: URL) throws -> URL { + guard #available(iOS 16.0, macOS 13.0, *) else { return source } + let files = FileManager.default + let attributes = try files.attributesOfItem(atPath: source.path) + let modified = (attributes[.modificationDate] as? Date)?.timeIntervalSinceReferenceDate ?? 0 + let key = "\(source.path)\n\(attributes[.systemFileNumber] ?? 0)\n\(attributes[.size] ?? 0)\n\(modified)" + let digest = SHA256.hash(data: Data(key.utf8)).map { String(format: "%02x", $0) }.joined() + let destination = cacheDirectory.appendingPathComponent(digest).appendingPathExtension("pdf") + if files.fileExists(atPath: destination.path) { return destination } + if unannotated.object(forKey: key as NSString) != nil { return source } + + guard let document = PDFDocument(url: source), !document.isLocked else { return source } + var hasAnnotations = false + for index in 0.. Bool { return false } + @objc + func preparePdfForRendering(_ fileUri: String, + resolver: @escaping RCTPromiseResolveBlock, + rejecter: @escaping RCTPromiseRejectBlock) { + Self.pdfRenderQueue.async { + autoreleasepool { + guard let source = Self.localFileURL(fileUri) else { + rejecter("E_INVALID_PATH", "Invalid PDF file URL", nil) + return + } + do { + let caches = try FileManager.default.url(for: .cachesDirectory, in: .userDomainMask, + appropriateFor: nil, create: true) + let directory = caches.appendingPathComponent("pdf-render-v1", isDirectory: true) + let rendered = try PDFRenderCopy.url(for: source, cacheDirectory: directory) + resolver(rendered == source ? fileUri : rendered.absoluteString) + } catch { + rejecter("E_PDF_RENDER_COPY", "Could not prepare the PDF annotations for rendering", error) + } + } + } + } + @objc func readImageForThumbnail(_ fileUri: String, resolver: @escaping RCTPromiseResolveBlock, diff --git a/scripts/catalogStore.test.mjs b/scripts/catalogStore.test.mjs index 317f0c2..53dfa78 100644 --- a/scripts/catalogStore.test.mjs +++ b/scripts/catalogStore.test.mjs @@ -14,15 +14,16 @@ test('catalog reload preserves composed thumbnail versions and still accepts old const catalog = parseCatalog(JSON.stringify({ version: 1, folders: [], notes: [ { id: 'old', title: 'Old note', thumbnailUri: 'old-ink-preview' }, - { id: 'new', title: 'Photo note', thumbnailUri: 'composed-cover', thumbnailVersion: 1 }, + { id: 'v1', title: 'Old cover', thumbnailUri: 'old-cover', thumbnailVersion: 1 }, + { id: 'new', title: 'PDF note', thumbnailUri: 'composed-cover', thumbnailVersion: 2 }, ], })); assert.equal(catalog.notes[0].thumbnailUri, 'old-ink-preview'); assert.equal(catalog.notes[0].thumbnailVersion, undefined); assert.equal(catalog.notes[1].thumbnailVersion, 1); const restored = parseCatalog(JSON.stringify(catalog)); - assert.equal(restored.notes[1].thumbnailUri, 'composed-cover'); - assert.equal(restored.notes[1].thumbnailVersion, 1); + assert.equal(restored.notes[2].thumbnailUri, 'composed-cover'); + assert.equal(restored.notes[2].thumbnailVersion, 2); }); function makeKv(initial = {}) { diff --git a/scripts/exportService.test.mjs b/scripts/exportService.test.mjs index fbfab44..2b837ce 100644 --- a/scripts/exportService.test.mjs +++ b/scripts/exportService.test.mjs @@ -24,7 +24,7 @@ function notebook(pages) { }; } -function harness(files = {}, { mergePdfFiles } = {}) { +function harness(files = {}, { mergePdfFiles, preparePdfForRendering } = {}) { const printCalls = []; const batchCalls = []; const fileReads = []; @@ -66,14 +66,14 @@ function harness(files = {}, { mergePdfFiles } = {}) { if (name === 'react-native') return { PixelRatio: { get: () => 2 } }; if (name === './pdfUtilsModule') { return { - PDFUtilsModule: mergePdfFiles === null - ? {} - : { - mergePdfFiles: async (uris) => { - mergeCalls.push(uris); - return mergePdfFiles ? mergePdfFiles(uris) : 'file:///merged.pdf'; - }, - }, + PDFUtilsModule: { + preparePdfForRendering, + mergePdfFiles: mergePdfFiles === null ? undefined : async (uris) => { + mergeCalls.push(uris); + return mergePdfFiles ? mergePdfFiles(uris) : 'file:///merged.pdf'; + }, + }, + resolvePdfRenderUri: async uri => uri ? preparePdfForRendering?.(uri) ?? uri : undefined, }; } if (name === '@mathnotes/mobile-ink') return mobileInk; @@ -256,3 +256,55 @@ test('base64 page rasters are embedded verbatim instead of escaped copies', () = const unsafe = buildPdfHtml([{ rasterUri: 'file:///a"b.png', images: [], textBoxes: [] }]); assert.ok(unsafe.includes('file:///a"b.png')); }); + +test('PDF export uses the prepared copy and preserves the source page index', async () => { + const source = 'file:///imported.pdf'; + const prepared = 'file:///cache/annotated.pdf'; + const h = harness({}, { preparePdfForRendering: async uri => { + assert.equal(uri, source); + return prepared; + } }); + + const result = await h.exportNotebookAsPdf({ + data: notebook([{ pageType: 'pdf', pdfPageNumber: 3 }]), + pdfBackgroundUri: source, + filename: 'marked-up', + }); + + assert.equal(result.ok, true); + assert.equal(h.batchCalls[0][5], prepared); + assert.equal(h.batchCalls[0][6][0], 2); +}); + +test('large exports regenerate annotated PDF pages while reusing other saved previews', async () => { + const oldPreview = 'file:///stale-preview.png'; + const plainPreview = 'data:image/png;base64,UEdF'; + const prepared = 'file:///cache/annotated.pdf'; + const h = harness({ [oldPreview]: 'T0xE' }, { preparePdfForRendering: async () => prepared }); + const data = notebook([ + { pageType: undefined, pdfPageNumber: 5, previewUri: oldPreview }, + ...Array.from({ length: 149 }, () => ({ previewUri: plainPreview })), + ]); + + const result = await h.exportNotebookAsPdf({ data, pdfBackgroundUri: 'file:///imported.pdf', filename: 'large' }); + + assert.equal(result.ok, true); + assert.equal(h.batchCalls.length, 1); + assert.equal(h.batchCalls[0][5], prepared); + assert.equal(h.batchCalls[0][6][0], 4); + assert.deepEqual(h.fileReads, []); + assert.doesNotMatch(h.printCalls[0].html, /T0xE/); + assert.match(h.printCalls[0].html, /UEdF/); +}); + +test('large unannotated PDF exports can still use saved previews', async () => { + const h = harness({}, { preparePdfForRendering: async uri => uri }); + const previewUri = 'data:image/png;base64,UERG'; + const data = notebook(Array.from({ length: 150 }, () => ({ pageType: 'pdf', previewUri }))); + + const result = await h.exportNotebookAsPdf({ data, pdfBackgroundUri: 'file:///imported.pdf', filename: 'large' }); + + assert.equal(result.ok, true); + assert.equal(h.batchCalls.length, 0); + assert.match(h.printCalls[0].html, /UERG/); +}); diff --git a/scripts/noteThumbnails.test.mjs b/scripts/noteThumbnails.test.mjs index daed3ef..a1fdd3d 100644 --- a/scripts/noteThumbnails.test.mjs +++ b/scripts/noteThumbnails.test.mjs @@ -9,6 +9,11 @@ const source = ts.transpileModule( { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }, ).outputText; +const importSource = ts.transpileModule( + readFileSync(new URL('../src/services/pdfImportService.ts', import.meta.url), 'utf8'), + { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }, +).outputText; + const ink = 'data:image/png;base64,aW5r'; const composite = 'data:image/png;base64,cGhvdG8='; const photo = { id: 'photo', type: 'image', sourceUri: 'file:///photo.jpg', x: 20, y: 30, width: 100, height: 80 }; @@ -31,14 +36,14 @@ function harness({ writeOk = true, renderFails = false, render = async () => com readBody: async () => { reads += 1; return { kind: readKind, data: structuredClone(stored) }; }, }; if (name === './thumbnailService') return { - THUMBNAIL_VERSION: 1, + THUMBNAIL_VERSION: 2, createPageThumbnail: async (...args) => { if (renderFails) throw new Error('Image decoder unavailable'); return render(...args); }, }; if (name === '../i18n') return { t: { common: { untitled: 'Untitled' } } }; - if (name === '../utils/id') return {}; + if (name === '../utils/id') return { noteId: () => 'imported' }; if (name === './catalogStore') return {}; if (name === './pdfStorage' || name === './imageInsertStorage') return {}; throw new Error('Unexpected import: ' + name); @@ -54,7 +59,7 @@ test('saving uses a composed cover without flattening overlays into the note bod const result = await h.saveNoteBody('note', original); assert.equal(result.ok, true); assert.equal(h.note().thumbnailUri, composite); - assert.equal(h.note().thumbnailVersion, 1); + assert.equal(h.note().thumbnailVersion, 2); assert.deepEqual(h.stored(), original); assert.equal(h.stored().pages[0].previewUri, ink); }); @@ -71,22 +76,23 @@ test('thumbnail failure cannot fail a note save or discard its images', async () assert.equal((await h.saveNoteBody('note', body())).ok, true); assert.equal(h.stored().pages[0].insertedElements[0].sourceUri, photo.sourceUri); assert.equal(h.note().thumbnailUri, ink); - assert.notEqual(h.note().thumbnailVersion, 1); + assert.notEqual(h.note().thumbnailVersion, 2); }); -test('legacy thumbnail repair leaves the note body and modification date untouched', async () => { +test('repairing a version-one cover leaves the note body and modification date untouched', async () => { const h = harness(); + await h.updateMetadata('note', { thumbnailVersion: 1, updatedAt: '2026-01-02' }); const before = structuredClone(h.stored()); assert.equal(await h.getNoteThumbnail(h.note()), composite); assert.equal(h.note().updatedAt, '2026-01-02'); - assert.equal(h.note().thumbnailVersion, 1); + assert.equal(h.note().thumbnailVersion, 2); assert.deepEqual(h.stored(), before); }); test('failed legacy thumbnail repair keeps the existing cover eligible for retry', async () => { const h = harness({ renderFails: true }); assert.equal(await h.getNoteThumbnail(h.note()), ink); - assert.notEqual(h.note().thumbnailVersion, 1); + assert.notEqual(h.note().thumbnailVersion, 2); }); test('image-only saves compose the first page even when no ink preview exists', async () => { @@ -126,7 +132,7 @@ for (const readKind of ['unreadable', 'missing']) { test('current thumbnail versions do not re-read or rewrite legacy bodies', async () => { const h = harness(); - await h.updateMetadata('note', { thumbnailVersion: 1, thumbnailUri: composite }); + await h.updateMetadata('note', { thumbnailVersion: 2, thumbnailUri: composite }); assert.equal(await h.getNoteThumbnail(h.note()), composite); assert.equal(h.reads(), 0); }); @@ -138,7 +144,7 @@ test('an old thumbnail repair cannot overwrite a newer note save', async () => { const h = harness({ render: () => new Promise(resolve => { finishRender = resolve; startRender(); }) }); const repairing = h.getNoteThumbnail(structuredClone(h.note())); await started; - await h.updateMetadata('note', { updatedAt: '2026-10-22', thumbnailUri: 'new-cover', thumbnailVersion: 1 }); + await h.updateMetadata('note', { updatedAt: '2026-10-22', thumbnailUri: 'new-cover', thumbnailVersion: 2 }); finishRender(composite); assert.equal(await repairing, 'new-cover'); assert.equal(h.note().thumbnailUri, 'new-cover'); @@ -165,3 +171,27 @@ test('duplicate card requests share a single repair of the same legacy cover', a assert.equal(await second, composite); assert.equal(h.reads(), 1); }); + +test('importing a PDF generates its first cover with the copied PDF already available', async () => { + const pdfUri = 'file:///notes/pdfs/imported.pdf'; + const h = harness({ render: async (_page, background, uri) => + background === 'pdf' && uri === pdfUri ? composite : null }); + const exports = {}; + vm.runInNewContext(importSource, { exports, require(name) { + if (name === './notesRepo') return h; + if (name === './lifecycleService') return { recordSuccessfulNoteSave: async () => {} }; + if (name === './pdfStorage') return { + importPdfFromUri: async () => ({ uri: pdfUri, name: 'Lecture.pdf', pageCount: 1 }), + createPdfNotebookData: () => ({ + version: '1.0', pages: [{ id: 'page-1', data: '', pageType: 'pdf', pdfPageNumber: 1 }], + }), + }; + throw new Error('Unexpected import: ' + name); + } }); + + const imported = await exports.createPdfNoteFromUri('file:///incoming/lecture.pdf'); + + assert.equal(imported.pdfUri, pdfUri); + assert.equal(imported.thumbnailUri, composite); + assert.equal(imported.thumbnailVersion, 2); +}); diff --git a/scripts/pdfRenderCopySmoke.swift b/scripts/pdfRenderCopySmoke.swift new file mode 100644 index 0000000..bfc2e79 --- /dev/null +++ b/scripts/pdfRenderCopySmoke.swift @@ -0,0 +1,85 @@ +import AppKit +import CoreGraphics +import Foundation +import PDFKit + +@main +struct PDFRenderCopySmoke { + static func main() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let cache = directory.appendingPathComponent("rendered") + let source = directory.appendingPathComponent("source.pdf") + let original = annotatedPDF() + try original.write(to: source) + + let rendered = try PDFRenderCopy.url(for: source, cacheDirectory: cache) + let page = CGPDFDocument(rendered as CFURL)!.page(at: 1)! + precondition(sample(page, x: 150, y: 150) == [255, 0, 0], "Markup is missing from the page content") + let printed = sample(page, x: 35, y: 35) + precondition(printed[0] < 80 && printed[1] < 80 && printed[2] > 200, "Original page content was lost: \(printed)") + precondition(sample(page, x: 150, y: 230) == [255, 255, 255], "Hidden annotation became visible") + precondition(page.getBoxRect(.mediaBox) == CGRect(x: 0, y: 0, width: 300, height: 300)) + let sourceAfter = try Data(contentsOf: source) + precondition(sourceAfter == original, "Original PDF was changed") + let cached = try PDFRenderCopy.url(for: source, cacheDirectory: cache) + precondition(cached == rendered, "Repeated opens should reuse the render copy") + + let plain = directory.appendingPathComponent("plain.pdf") + try basePDF().write(to: plain) + let plainResult = try PDFRenderCopy.url(for: plain, cacheDirectory: cache) + precondition(plainResult == plain, "Unannotated PDFs should use the original") + try basePDF().write(to: source, options: .atomic) + let replaced = try PDFRenderCopy.url(for: source, cacheDirectory: cache) + precondition(replaced == source, "A replaced source must not reuse an old render copy") + } + + static func basePDF() -> Data { + let data = NSMutableData() + var box = CGRect(x: 0, y: 0, width: 300, height: 300) + let consumer = CGDataConsumer(data: data as CFMutableData)! + let context = CGContext(consumer: consumer, mediaBox: &box, nil)! + context.beginPDFPage(nil) + context.setFillColor(CGColor(red: 0, green: 0, blue: 1, alpha: 1)) + context.fill(CGRect(x: 20, y: 20, width: 30, height: 30)) + context.endPDFPage() + context.closePDF() + return data as Data + } + + static func annotatedPDF() -> Data { + let document = PDFDocument(data: basePDF())! + let page = document.page(at: 0)! + page.addAnnotation(inkAnnotation(y: 100)) + let hidden = inkAnnotation(y: 180) + hidden.shouldDisplay = false + hidden.shouldPrint = true + page.addAnnotation(hidden) + return document.dataRepresentation()! + } + + static func inkAnnotation(y: CGFloat) -> PDFAnnotation { + let annotation = PDFAnnotation(bounds: CGRect(x: 100, y: y, width: 100, height: 100), forType: .ink, withProperties: nil) + annotation.color = .red + let border = PDFBorder() + border.lineWidth = 10 + annotation.border = border + let path = NSBezierPath() + path.move(to: CGPoint(x: 10, y: 50)) + path.line(to: CGPoint(x: 90, y: 50)) + annotation.add(path) + return annotation + } + + static func sample(_ page: CGPDFPage, x: Int, y: Int) -> [UInt8] { + let context = CGContext(data: nil, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.setFillColor(CGColor(gray: 1, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: 1, height: 1)) + context.translateBy(x: -CGFloat(x), y: -CGFloat(y)) + context.drawPDFPage(page) + return Array(UnsafeBufferPointer(start: context.data!.assumingMemoryBound(to: UInt8.self), count: 3)) + } +} diff --git a/scripts/thumbnailRendering.test.mjs b/scripts/thumbnailRendering.test.mjs index 9f0e5ff..c243b91 100644 --- a/scripts/thumbnailRendering.test.mjs +++ b/scripts/thumbnailRendering.test.mjs @@ -44,14 +44,19 @@ const red = solid('#ff0000'); const blue = solid('#0000ff'); const black = solid('#000000'); -function harness({ nativeFails = false, nativeParagraphBuilder = false, nativeImage } = {}) { +function harness({ + nativeFails = false, nativeParagraphBuilder = false, nativeImage, density = 2, + preparePdfForRendering, render = () => white, +} = {}) { const exports = {}; const nativeCalls = []; vm.runInNewContext(source, { exports, require(name) { if (name === '../utils/promiseQueue') return { createPromiseQueue }; - if (name === './pdfUtilsModule') return { PDFUtilsModule: nativeImage ? { - readImageForThumbnail: nativeImage, - } : undefined }; + if (name === 'react-native') return { PixelRatio: { get: () => density } }; + if (name === './pdfUtilsModule') return { + PDFUtilsModule: nativeImage ? { readImageForThumbnail: nativeImage } : undefined, + resolvePdfRenderUri: async uri => uri ? preparePdfForRendering?.(uri) ?? uri : undefined, + }; if (name === '@shopify/react-native-skia') return { Skia: nativeParagraphBuilder ? { ...Skia, ParagraphBuilder: { Make: (...args) => { const builder = Skia.ParagraphBuilder.Make(...args); @@ -62,7 +67,7 @@ function harness({ nativeFails = false, nativeParagraphBuilder = false, nativeIm if (name === '@mathnotes/mobile-ink') return { batchExportPages: async (...args) => { nativeCalls.push(args); if (nativeFails) throw new Error('Native renderer unavailable'); - return [white]; + return [render(...args)]; } }; throw new Error('Unexpected import: ' + name); } }); @@ -192,6 +197,64 @@ test('PDF thumbnails without a preview request the correct background page', asy assert.equal(h.nativeCalls[0][6][0], 2); }); +const savedPdfInk = '{"pages":{"0":"c2F2ZWQ="}}'; +const preparedPdf = 'file:///cache/lecture.pdf'; + +function pdfRaster(payloads, _types, width, height, scale, uri) { + // Mirror the native raster bounds and fixed device-pixel ink coordinates. + const surface = Skia.Surface.Make(Math.floor(width * scale), Math.floor(height * scale)); + const canvas = surface.getCanvas(); + canvas.clear(Skia.Color(uri === preparedPdf ? '#0000ff' : '#ffffff')); + canvas.scale(scale, scale); + const paint = Skia.Paint(); + paint.setColor(Skia.Color('#000000')); + if (payloads[0] === savedPdfInk) { + canvas.drawRect(Skia.XYWHRect(400, 800, 80, 24), paint); + canvas.drawRect(Skia.XYWHRect(400, 2120, 80, 2), paint); + } + const image = surface.makeImageSnapshot(); + try { + return 'data:image/png;base64,' + image.encodeToBase64(); + } finally { + image.dispose(); + paint.dispose(); + surface.dispose(); + } +} + +test('PDF covers use saved ink over the prepared background even when a preview exists', async () => { + const h = harness({ preparePdfForRendering: async () => preparedPdf, render: pdfRaster }); + const result = await h.createPageThumbnail( + page([], { data: savedPdfInk, previewUri: red, pageType: 'pdf', pdfPageNumber: 3 }), + 'pdf', 'file:///lecture.pdf', + ); + assert.deepEqual(pixel(result, 10, 10), [0, 0, 255, 255]); + assert.deepEqual(pixel(result, 110, 202), [0, 0, 0, 255]); + assert.deepEqual(pixel(result, 210, 405), [0, 0, 255, 255]); + assert.equal(h.nativeCalls[0][6][0], 2); +}); + +test('a PDF cover retains bottom-edge ink and composites overlays at page coordinates', async () => { + const h = harness({ preparePdfForRendering: async () => preparedPdf, render: pdfRaster }); + const result = await h.createPageThumbnail( + page([photo()], { data: savedPdfInk, previewUri: red, pageType: 'pdf' }), + 'pdf', 'file:///lecture.pdf', + ); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); + assert.deepEqual(pixel(result, 180, 100), [0, 0, 255, 255]); + const bottom = pixel(result, 110, 530); + assert.equal(bottom[3], 255); + assert.ok(bottom[2] < 200, 'Expected ink in the final thumbnail row'); +}); + +test('a PDF with no handwriting or stored preview still produces a cover', async () => { + const h = harness({ preparePdfForRendering: async () => preparedPdf, render: pdfRaster }); + const result = await h.createPageThumbnail(page([], { data: '', previewUri: undefined, pageType: 'pdf' }), + 'pdf', 'file:///lecture.pdf'); + assert.ok(result); + assert.deepEqual(pixel(result, 10, 10), [0, 0, 255, 255]); +}); + test('typed text is included and clipped to its original text box', async () => { const result = await harness().createPageThumbnail(page([], { textBoxes: [{ id: 'caption', content: 'Visible text\nSecond line\nHidden line', color: '#000000', diff --git a/src/services/catalogStore.ts b/src/services/catalogStore.ts index 81e1107..9e03cdb 100644 --- a/src/services/catalogStore.ts +++ b/src/services/catalogStore.ts @@ -123,7 +123,8 @@ function normalizeNote(value: unknown): NoteMetadata | null { backgroundType, pdfUri: isNonEmptyString(v.pdfUri) ? v.pdfUri : null, thumbnailUri: isNonEmptyString(v.thumbnailUri) ? v.thumbnailUri : null, - ...(v.thumbnailVersion === 1 ? { thumbnailVersion: 1 } : {}), + ...(typeof v.thumbnailVersion === 'number' && Number.isSafeInteger(v.thumbnailVersion) && v.thumbnailVersion > 0 + ? { thumbnailVersion: v.thumbnailVersion } : {}), }; } diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 3a6a5a7..9caae79 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -11,7 +11,7 @@ import { PDF_PAGE_WIDTH, type PrintablePage, } from './pdfExportHtml'; -import { PDFUtilsModule } from './pdfUtilsModule'; +import { PDFUtilsModule, resolvePdfRenderUri } from './pdfUtilsModule'; const EXPORT_SCALE = 1.0; const DEFAULT_NATIVE_BATCH_SIZE = 8; @@ -43,6 +43,10 @@ function getPdfPageIndex( : pageIndex; } +function getPageType(page: SerializedNotebookData['pages'][number], pdfBackgroundUri?: string): string { + return page.pageType ?? (pdfBackgroundUri ? 'pdf' : 'plain'); +} + async function toPrintableImageUri(uri: string | undefined): Promise { if (!uri) return null; if (uri.startsWith('data:image/')) return uri; @@ -131,7 +135,7 @@ async function renderNativePagesInChunks(options: { pageIndexes: number[]; exportWidth: number; exportHeight: number; - pdfBackgroundUri?: string | null; + pdfBackgroundUri?: string; batchSize: number; }): Promise> { const { @@ -149,11 +153,11 @@ async function renderNativePagesInChunks(options: { const chunkPages = chunkPageIndexes.map((pageIndex) => data.pages[pageIndex]); const pngUris = await batchExportPages( chunkPages.map((p) => p?.data ?? '{"pages":{}}'), - chunkPages.map((p) => p?.pageType ?? 'plain'), + chunkPages.map((p) => getPageType(p, pdfBackgroundUri)), exportWidth, exportHeight, EXPORT_SCALE, - pdfBackgroundUri ?? undefined, + pdfBackgroundUri, chunkPages.map((page, offset) => getPdfPageIndex(page, chunkPageIndexes[offset]), ), @@ -179,16 +183,21 @@ async function collectExportImages(options: { pageIndexes: number[]; exportWidth: number; exportHeight: number; - pdfBackgroundUri?: string | null; + pdfBackgroundUri?: string; + hasBakedAnnotations: boolean; }): Promise { - const { data, pageIndexes, exportWidth, exportHeight, pdfBackgroundUri } = options; + const { data, pageIndexes, exportWidth, exportHeight, pdfBackgroundUri, hasBakedAnnotations } = options; const images = new Map(); const nativePageIndexes: number[] = []; const preferStoredPreviews = data.pages.length >= LARGE_NOTEBOOK_PREVIEW_THRESHOLD; if (preferStoredPreviews) { for (const pageIndex of pageIndexes) { - const previewUri = await toPrintableImageUri(data.pages[pageIndex]?.previewUri); + const page = data.pages[pageIndex]; + // Saved previews may have been captured before the annotations were baked in. + const previewUri = hasBakedAnnotations && getPageType(page, pdfBackgroundUri) === 'pdf' + ? null + : await toPrintableImageUri(page.previewUri); if (previewUri) { images.set(pageIndex, previewUri); } else { @@ -246,6 +255,8 @@ export async function exportNotebookAsPdf( const passUris: string[] = []; let mergedUri: string | null = null; try { + const renderPdfUri = await resolvePdfRenderUri(pdfBackgroundUri); + const hasBakedAnnotations = renderPdfUri !== undefined && renderPdfUri !== pdfBackgroundUri; const pixelRatio = PixelRatio.get(); const nativeScale = Number.isFinite(pixelRatio) && pixelRatio > 0 ? pixelRatio : 1; const exportWidth = Math.round(NOTE_PAGE_WIDTH * nativeScale); @@ -262,7 +273,8 @@ export async function exportNotebookAsPdf( pageIndexes, exportWidth, exportHeight, - pdfBackgroundUri, + pdfBackgroundUri: renderPdfUri, + hasBakedAnnotations, }); if (rasterUris.length !== pageIndexes.length) { return { diff --git a/src/services/pdfImportService.ts b/src/services/pdfImportService.ts index f33300a..92d6343 100644 --- a/src/services/pdfImportService.ts +++ b/src/services/pdfImportService.ts @@ -55,15 +55,15 @@ async function createPdfNote( } const title = cleanTitle(options.title) || cleanTitle(result.name.replace(/\.pdf$/i, '')); + await setNoteBackground(meta.id, 'pdf', result.uri); const body = createPdfNotebookData(result.pageCount); const saveResult = await saveNoteBody(meta.id, body); if (!saveResult.ok) { throw new Error('Imported PDF note body could not be saved.'); } - const withBackground = await setNoteBackground(meta.id, 'pdf', result.uri); const finalMetadata = title ? await renameNote(meta.id, title) - : withBackground; + : saveResult.metadata; await recordSuccessfulNoteSave(meta.id); return finalMetadata; } catch (error) { diff --git a/src/services/pdfUtilsModule.ts b/src/services/pdfUtilsModule.ts index 12ff948..65f2036 100644 --- a/src/services/pdfUtilsModule.ts +++ b/src/services/pdfUtilsModule.ts @@ -5,7 +5,13 @@ export type PDFUtilsModuleType = { getPageCount?: (filePath: string) => Promise; copySecurityScopedFileToTmp?: (sourceUrl: string) => Promise; mergePdfFiles?: (fileUris: string[]) => Promise; + preparePdfForRendering?: (fileUri: string) => Promise; readImageForThumbnail?: (fileUri: string) => Promise; }; export const PDFUtilsModule = NativeModules.PDFUtilsModule as PDFUtilsModuleType | undefined; + +export async function resolvePdfRenderUri(uri: string | null | undefined): Promise { + if (!uri) return undefined; + return PDFUtilsModule?.preparePdfForRendering?.(uri) ?? uri; +} diff --git a/src/services/thumbnailService.ts b/src/services/thumbnailService.ts index 11824d5..8a4adc2 100644 --- a/src/services/thumbnailService.ts +++ b/src/services/thumbnailService.ts @@ -1,12 +1,14 @@ import { ClipOp, Skia, type SkImage } from '@shopify/react-native-skia'; import { batchExportPages, type InsertedElement, type NotebookPage } from '@mathnotes/mobile-ink'; +import { PixelRatio } from 'react-native'; import { createPromiseQueue } from '../utils/promiseQueue'; -import { PDFUtilsModule } from './pdfUtilsModule'; +import { PDFUtilsModule, resolvePdfRenderUri } from './pdfUtilsModule'; -export const THUMBNAIL_VERSION = 1; +export const THUMBNAIL_VERSION = 2; const PAGE_WIDTH = 820; const PAGE_HEIGHT = 1061; const SCALE = 0.5; +const THUMBNAIL_HEIGHT = Math.ceil(PAGE_HEIGHT * SCALE); const renderQueue = createPromiseQueue(); function finite(value: number | undefined, fallback: number): number { @@ -60,18 +62,32 @@ async function composePageThumbnail( .sort((a, b) => finite(a.zIndex, 1) - finite(b.zIndex, 1) || finite(a.zIndex, 0) - finite(b.zIndex, 0)); const textBoxes = (page.textBoxes ?? []).filter(box => Boolean(box.content)); - if (!elements.length && !textBoxes.length) return page.previewUri ?? null; + const pageType = page.pageType ?? backgroundType; + const isPdf = pageType === 'pdf'; + const hasOverlays = elements.length > 0 || textBoxes.length > 0; + if (!isPdf && !hasOverlays) return page.previewUri ?? null; - let raster = page.previewUri; + // Round the PDF raster up to a full thumbnail row so the native renderer cannot clip the page edge. + const rasterHeight = isPdf ? THUMBNAIL_HEIGHT / SCALE : PAGE_HEIGHT; + // The editor preview can contain stale PDF content or mis-scaled ink. + let raster = isPdf ? undefined : page.previewUri; if (!raster) { + const renderPdfUri = isPdf ? await resolvePdfRenderUri(pdfUri) : undefined; + if (isPdf && !renderPdfUri) throw new Error('Thumbnail PDF background is missing'); + const density = isPdf ? PixelRatio.get() : 1; + const nativeScale = Number.isFinite(density) && density > 0 ? density : 1; + // Mobile Ink stores ink in device pixels. + const width = Math.ceil(PAGE_WIDTH * nativeScale); + const height = Math.ceil(rasterHeight * nativeScale); [raster] = await batchExportPages( - [page.data ?? '{"pages":{}}'], [page.pageType ?? backgroundType], PAGE_WIDTH, PAGE_HEIGHT, SCALE, - pdfUri ?? undefined, [page.pdfPageNumber ? page.pdfPageNumber - 1 : pageIndex], + [page.data || '{"pages":{}}'], [pageType], width, height, SCALE / nativeScale, + renderPdfUri, [page.pdfPageNumber ? page.pdfPageNumber - 1 : pageIndex], ); if (!raster) throw new Error('Thumbnail background could not be rendered'); } + if (!hasOverlays) return raster; - const surface = Skia.Surface.Make(PAGE_WIDTH * SCALE, Math.ceil(PAGE_HEIGHT * SCALE)); + const surface = Skia.Surface.Make(PAGE_WIDTH * SCALE, THUMBNAIL_HEIGHT); if (!surface) throw new Error('Thumbnail surface could not be allocated'); const paint = Skia.Paint(); try { @@ -81,7 +97,7 @@ async function composePageThumbnail( const base = await decodeImage(raster); try { canvas.drawImageRect(base, Skia.XYWHRect(0, 0, base.width(), base.height()), - Skia.XYWHRect(0, 0, PAGE_WIDTH, PAGE_HEIGHT), paint); + Skia.XYWHRect(0, 0, PAGE_WIDTH, rasterHeight), paint); } finally { base.dispose(); }