From 2b790b26fb5f752cbac2c21d24ebce6c86622608 Mon Sep 17 00:00:00 2001 From: Wei Lin Date: Sun, 8 Nov 2026 23:08:35 +0800 Subject: [PATCH] Add opt-in low-memory XLSX conversion for .NET Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- documents/README.nuget.cli.md | 10 + documents/README.nuget.md | 49 +++ scripts/Measure-DotNet-XlsxMemory.ps1 | 109 ++++++ src/MiniPdf.Cli/Program.cs | 6 + src/MiniPdf/ExcelReader.cs | 93 ++++-- src/MiniPdf/ExcelToPdfConverter.cs | 85 +++-- src/MiniPdf/FileBackedList.cs | 309 +++++++++++++++++ src/MiniPdf/LowMemoryWorkbook.cs | 166 +++++++++ src/MiniPdf/MiniPdf.cs | 27 +- src/MiniPdf/PdfDocument.cs | 26 +- src/MiniPdf/PdfWriter.cs | 11 +- tests/MiniPdf.Tests/LowMemoryXlsxTests.cs | 390 ++++++++++++++++++++++ 12 files changed, 1207 insertions(+), 74 deletions(-) create mode 100644 scripts/Measure-DotNet-XlsxMemory.ps1 create mode 100644 src/MiniPdf/FileBackedList.cs create mode 100644 src/MiniPdf/LowMemoryWorkbook.cs create mode 100644 tests/MiniPdf.Tests/LowMemoryXlsxTests.cs diff --git a/documents/README.nuget.cli.md b/documents/README.nuget.cli.md index 17eee928..92a892eb 100644 --- a/documents/README.nuget.cli.md +++ b/documents/README.nuget.cli.md @@ -33,6 +33,9 @@ minipdf report.docx --fonts ./Fonts # Compress PDF content streams for large outputs minipdf data.xlsx --compress +# Reduce memory for large simple XLSX tables using temporary disk storage +minipdf data.xlsx --low-memory --compress + # Render a bounded Excel preview minipdf data.xlsx --max-rows 200 --max-columns 20 --compress @@ -45,12 +48,19 @@ minipdf data.xlsx --max-rows 200 --max-columns 20 --compress | `minipdf ` | Convert `.xlsx` / `.docx` / `.pptx` to PDF | | `minipdf convert -o ` | Convert with explicit output path | | `minipdf convert --compress` | Compress PDF content streams | +| `minipdf data.xlsx --low-memory` | Use opt-in disk-backed XLSX conversion | | `minipdf data.xlsx --max-rows --max-columns ` | Render a bounded Excel preview | | `minipdf --version` | Show version | | `minipdf --help` | Show help | ## Features +`--low-memory` preserves the default conversion path when omitted. It requires +writable OS temporary storage and supports simple XLSX tables only. Unsupported +worksheet features and options produce an explicit error; there is no automatic +high-memory fallback. See the [library guide](README.nuget.md#low-memory-xlsx-conversion-opt-in) +for supported features and limitations. + - No COM, no Office installation, no LibreOffice — runs anywhere .NET runs - Single command, zero config - Custom font registration for headless / container environments diff --git a/documents/README.nuget.md b/documents/README.nuget.md index 8eec6e9a..c55887d5 100644 --- a/documents/README.nuget.md +++ b/documents/README.nuget.md @@ -88,6 +88,55 @@ Sheet selection and Excel layout options apply to XLSX input. `PageSize` and `Margins` apply to DOCX input and use PDF points (72 points = 1 inch). `Compress` controls PDF content-stream compression for every supported format. +### Low-memory XLSX conversion (opt-in) + +```csharp +using var input = File.OpenRead("large-table.xlsx"); +using var output = File.Create("large-table.pdf"); +MiniPdf.ConvertToPdf(input, output, new MiniPdfConversionOptions +{ + LowMemory = true, + Compress = true, +}); +``` + +`LowMemory` defaults to `false`; the existing conversion path is unchanged. +The opt-in mode streams worksheet XML to temporary storage, keeps rows and +completed pages on disk, and writes PDF content one page at a time. It reuses +the existing layout and font embedding logic, including Unicode text, cell +styles, number formats, column widths, wrapping and horizontal pagination. +Both file and stream overloads support it. Non-seekable inputs are spooled to +disk instead of memory. Temporary files are deleted when conversion finishes +or fails. + +This first version supports simple tables. It throws `NotSupportedException` +rather than silently falling back for formulas, merged cells, conditional +formatting, drawings/charts/images, Excel tables, hyperlinks, headers/footers, +worksheet extensions, custom/hidden row heights, vertical centering, print +areas/titles, and fit-to-height pagination. Worksheet feature restrictions +apply to the entire workbook, including unselected sheets. `MaxRows`, +`MaxColumns` and `RowsPerPage` are not supported in this mode. + +Use a file or output stream to avoid retaining the final PDF. Overloads that +return `byte[]` still allocate the complete output. Writable OS temporary +storage is required; disk use increases, and extra I/O can slow conversion. This is disk-backed +low-memory conversion, not a single-pass/no-disk pipeline. Offset indexes, +workbook/style metadata and font resources still consume memory proportional +to their size, and a single unusually large row or page can increase the peak. +This mode is not suitable for hosts without writable temporary storage. + +To measure the .NET implementation locally (PowerShell 7 and Python required): + +```powershell +.\scripts\Measure-DotNet-XlsxMemory.ps1 -RowCounts 20000,60000 +``` + +The script runs each mode in a separate process, verifies identical PDF hashes, +records peak working sets and elapsed times under `artifacts/xlsx-low-memory`, +and requires at least a 30% peak working-set reduction on the largest generated +eight-column case. This is a regression benchmark, not a guarantee for every +workbook. + ### Custom Fonts Register TrueType `.ttf` or `.ttc` fonts once at application startup when the diff --git a/scripts/Measure-DotNet-XlsxMemory.ps1 b/scripts/Measure-DotNet-XlsxMemory.ps1 new file mode 100644 index 00000000..eab47e28 --- /dev/null +++ b/scripts/Measure-DotNet-XlsxMemory.ps1 @@ -0,0 +1,109 @@ +#requires -Version 7.0 +<# +.SYNOPSIS + Measures isolated-process peak working sets and verifies identical XLSX PDF output. +.EXAMPLE + .\scripts\Measure-DotNet-XlsxMemory.ps1 -RowCounts 20000,60000 +#> +param( + [ValidateNotNullOrEmpty()] + [int[]]$RowCounts = @(20000, 60000) +) + +$ErrorActionPreference = "Stop" +$RepoRoot = Split-Path -Parent $PSScriptRoot +$OutputDir = Join-Path $RepoRoot "artifacts\xlsx-low-memory\memory" +$Project = Join-Path $RepoRoot "src\MiniPdf.Cli\MiniPdf.Cli.csproj" +$Cli = Join-Path $RepoRoot "src\MiniPdf.Cli\bin\Release\net9.0\MiniPdf.Cli.dll" +$Dotnet = (Get-Command dotnet -ErrorAction Stop).Source +if ($RowCounts | Where-Object { $_ -lt 1 -or $_ -gt 1048576 }) { + throw "RowCounts must be between 1 and 1048576." +} +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null +& $Dotnet build $Project -c Release --no-restore *> (Join-Path $OutputDir "build.log") +if ($LASTEXITCODE -ne 0) { throw "CLI build failed. See $OutputDir\build.log." } + +$Generator = @' +import io, pathlib, sys, zipfile +directory = pathlib.Path(sys.argv[1]) +ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +for count in map(int, sys.argv[2:]): + with zipfile.ZipFile(directory / f"table-{count}.xlsx", "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("xl/workbook.xml", f'') + with io.TextIOWrapper(z.open("xl/sharedStrings.xml", "w"), encoding="utf-8") as out: + out.write(f'') + for i in range(1, count+1): + out.write(f'Record {i:08d}') + out.write('') + with io.TextIOWrapper(z.open("xl/worksheets/sheet1.xml", "w"), encoding="utf-8") as out: + out.write(f'') + for i in range(1, count+1): + out.write(f'{i-1}') + for col in "BCDEFGH": + out.write(f'{i*10+ord(col)}') + out.write('') + out.write('') +'@ +$Generator | python - $OutputDir @RowCounts +if ($LASTEXITCODE -ne 0) { throw "Generating the memory benchmark workbooks failed." } + +$Results = @() +foreach ($Rows in ($RowCounts | Sort-Object -Unique)) { + foreach ($LowMemory in @($false, $true)) { + $Mode = if ($LowMemory) { "low-memory" } else { "default" } + $Pdf = Join-Path $OutputDir "table-$Rows-$Mode.pdf" + $Info = [System.Diagnostics.ProcessStartInfo]::new($Dotnet) + $Info.UseShellExecute = $false + $Info.RedirectStandardOutput = $true + $Info.RedirectStandardError = $true + foreach ($Argument in @($Cli, (Join-Path $OutputDir "table-$Rows.xlsx"), "-o", $Pdf)) { + $Info.ArgumentList.Add($Argument) + } + if ($LowMemory) { $Info.ArgumentList.Add("--low-memory") } + $Process = [System.Diagnostics.Process]::new() + $Process.StartInfo = $Info + $Timer = [System.Diagnostics.Stopwatch]::StartNew() + $Peak = 0L + $Started = $false + try { + $Started = $Process.Start() + if (-not $Started) { throw "Could not start the conversion process." } + $Stdout = $Process.StandardOutput.ReadToEndAsync() + $Stderr = $Process.StandardError.ReadToEndAsync() + while (-not $Process.HasExited) { + $Process.Refresh() + $Peak = [Math]::Max($Peak, $Process.PeakWorkingSet64) + Start-Sleep -Milliseconds 20 + } + $Process.WaitForExit() + $Timer.Stop() + $Output = $Stdout.GetAwaiter().GetResult() + $ErrorOutput = $Stderr.GetAwaiter().GetResult() + [System.IO.File]::WriteAllText((Join-Path $OutputDir "table-$Rows-$Mode.log"), $Output + $ErrorOutput) + if ($Process.ExitCode -ne 0) { throw "Conversion failed: $ErrorOutput" } + if ($Peak -eq 0) { throw "No memory sample was captured." } + $Result = [pscustomobject]@{ + Rows = $Rows + Columns = 8 + Mode = $Mode + PeakWorkingSetBytes = $Peak + Seconds = [Math]::Round($Timer.Elapsed.TotalSeconds, 3) + PdfBytes = (Get-Item -LiteralPath $Pdf).Length + Sha256 = (Get-FileHash -LiteralPath $Pdf -Algorithm SHA256).Hash + } + $Results += $Result + $Result | Format-List + } + finally { + if ($Started -and -not $Process.HasExited) { Stop-Process -Id $Process.Id } + $Process.Dispose() + } + } + $Pair = @($Results | Where-Object Rows -eq $Rows) + if ($Pair[0].Sha256 -ne $Pair[1].Sha256) { throw "PDF output differs for $Rows rows." } +} +$Results | ConvertTo-Json | Set-Content (Join-Path $OutputDir "results.json") -Encoding utf8 +$Largest = @($Results | Where-Object Rows -eq ($RowCounts | Measure-Object -Maximum).Maximum) +$Ratio = $Largest[1].PeakWorkingSetBytes / $Largest[0].PeakWorkingSetBytes +if ($Ratio -gt 0.7) { throw "LowMemory must reduce peak working set by at least 30% on the largest case; ratio was $Ratio." } +Write-Host ("Largest-case peak working-set reduction: {0:P1}" -f (1 - $Ratio)) diff --git a/src/MiniPdf.Cli/Program.cs b/src/MiniPdf.Cli/Program.cs index 982b1eae..ef8cefc4 100644 --- a/src/MiniPdf.Cli/Program.cs +++ b/src/MiniPdf.Cli/Program.cs @@ -32,6 +32,7 @@ private static int RunConvert(ReadOnlySpan args) string[]? sheets = null; int[]? sheetIndexes = null; bool compress = false; + bool lowMemory = false; int? maxRows = null; int? maxColumns = null; bool? landscape = null; @@ -57,6 +58,9 @@ private static int RunConvert(ReadOnlySpan args) case "--compress": compress = true; break; + case "--low-memory": + lowMemory = true; + break; case "--max-rows": if (i + 1 >= args.Length || !TryParsePositiveInt(args[++i], out var parsedMaxRows)) { @@ -169,6 +173,7 @@ private static int RunConvert(ReadOnlySpan args) Sheets = sheets, SheetIndexes = sheetIndexes, Compress = compress, + LowMemory = lowMemory, MaxRows = maxRows, MaxColumns = maxColumns, Landscape = landscape, @@ -259,6 +264,7 @@ private static void ShowConvertHelp() " --fonts Directory of .ttf/.ttc fonts to register", " --sheets Comma-separated Excel sheet names or 1-based indexes to render", " --compress Compress PDF content streams with FlateDecode", + " --low-memory Use disk-backed XLSX rows and pages (simple tables only)", " --max-rows Render only the first n rows from each Excel sheet or print area", " --max-columns Render only the first n columns from each Excel sheet or print area", " --landscape Render Excel sheets in landscape orientation", diff --git a/src/MiniPdf/ExcelReader.cs b/src/MiniPdf/ExcelReader.cs index fc28e32c..6d2dcbbb 100644 --- a/src/MiniPdf/ExcelReader.cs +++ b/src/MiniPdf/ExcelReader.cs @@ -34,14 +34,17 @@ private sealed record DxfStyleInfo /// Reads all sheets from an Excel file and returns their data as a list of sheets, /// where each sheet is a list of rows, and each row is a list of cell values. /// - internal static List ReadSheets(Stream stream, System.Globalization.CultureInfo? culture = null) + internal static List ReadSheets(Stream stream, System.Globalization.CultureInfo? culture = null, + LowMemoryWorkbook? lowMemory = null) { var sheets = new List(); using var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); // Read shared strings table - var (sharedStrings, boldPrefixLengths) = ReadSharedStrings(archive); + var (sharedStrings, boldPrefixLengths) = lowMemory == null + ? ReadSharedStrings(archive) + : ((IList)lowMemory.SharedStrings, lowMemory.BoldPrefixLengths); // Read theme colors and styles var themeColors = ReadThemeColors(archive); @@ -79,7 +82,7 @@ internal static List ReadSheets(Stream stream, System.Globalization. if (entry == null) continue; var hyperlinks = ReadWorksheetHyperlinks(archive, entry.FullName); - var rows = ReadSheet(entry, sharedStrings, boldPrefixLengths, fontStyles, fillColors, borders, numberFormats, cellXfFontIndices, cellXfFillIndices, cellXfNumFmtIds, cellXfAlignments, cellXfVerticalAlignments, cellXfTextRotations, cellXfBorderIndices, cellXfWrapTexts, cellXfIndents, hyperlinks, culture); + var rows = ReadSheet(entry, sharedStrings, boldPrefixLengths, fontStyles, fillColors, borders, numberFormats, cellXfFontIndices, cellXfFillIndices, cellXfNumFmtIds, cellXfAlignments, cellXfVerticalAlignments, cellXfTextRotations, cellXfBorderIndices, cellXfWrapTexts, cellXfIndents, hyperlinks, culture, lowMemory); var images = ReadSheetImages(archive, entry.FullName); var drawingShapes = ReadSheetShapes(archive, entry.FullName, themeColors); var (colWidths, defaultColWidth) = ReadColumnWidths(entry); @@ -108,7 +111,7 @@ internal static List ReadSheets(Stream stream, System.Globalization. if (entry != null) { var hyperlinks = ReadWorksheetHyperlinks(archive, entry.FullName); - var rows = ReadSheet(entry, sharedStrings, boldPrefixLengths, fontStyles, fillColors, borders, numberFormats, cellXfFontIndices, cellXfFillIndices, cellXfNumFmtIds, cellXfAlignments, cellXfVerticalAlignments, cellXfTextRotations, cellXfBorderIndices, cellXfWrapTexts, cellXfIndents, hyperlinks, culture); + var rows = ReadSheet(entry, sharedStrings, boldPrefixLengths, fontStyles, fillColors, borders, numberFormats, cellXfFontIndices, cellXfFillIndices, cellXfNumFmtIds, cellXfAlignments, cellXfVerticalAlignments, cellXfTextRotations, cellXfBorderIndices, cellXfWrapTexts, cellXfIndents, hyperlinks, culture, lowMemory); var images = ReadSheetImages(archive, entry.FullName); var (colWidths, defaultColWidth) = ReadColumnWidths(entry); var mergedCells = ReadMergedCells(entry); @@ -159,7 +162,7 @@ internal static List ReadSheets(Stream stream, System.Globalization. return sheets; } - private static (List Strings, Dictionary BoldPrefixLengths) ReadSharedStrings(ZipArchive archive) + private static (IList Strings, Dictionary BoldPrefixLengths) ReadSharedStrings(ZipArchive archive) { var strings = new List(); var boldPrefixLengths = new Dictionary(); @@ -172,26 +175,24 @@ private static (List Strings, Dictionary BoldPrefixLengths) Re foreach (var si in doc.Descendants(ns + "si")) { - var runs = si.Elements(ns + "r").ToList(); - if (runs.Count >= 2) - { - // Rich text: check if the first run is bold and subsequent runs are not. - var firstRpr = runs[0].Element(ns + "rPr"); - var firstBold = firstRpr?.Element(ns + "b") != null; - if (firstBold) - { - var firstText = string.Concat(runs[0].Descendants(ns + "t").Select(t => t.Value)); - if (firstText.Length > 0) - boldPrefixLengths[strings.Count] = firstText.Length; - } - } - var text = string.Concat(si.Descendants(ns + "t").Select(t => t.Value)); + var (text, boldPrefix) = ReadSharedString(si); + if (boldPrefix > 0) boldPrefixLengths[strings.Count] = boldPrefix; strings.Add(text); } return (strings, boldPrefixLengths); } + internal static (string Text, int BoldPrefix) ReadSharedString(XElement item) + { + var ns = item.Name.Namespace; + var runs = item.Elements(ns + "r").ToList(); + var prefix = runs.Count >= 2 && runs[0].Element(ns + "rPr")?.Element(ns + "b") != null + ? string.Concat(runs[0].Descendants(ns + "t").Select(text => text.Value)).Length + : 0; + return (string.Concat(item.Descendants(ns + "t").Select(text => text.Value)), prefix); + } + private static (List Sheets, Dictionary PrintAreas, Dictionary PrintTitleRows) ReadWorkbook(ZipArchive archive) { var result = new List(); @@ -489,7 +490,7 @@ private static Dictionary ReadTableStyleBorders(ZipArchi /// to cells within each table range that don't already have explicit borders. /// private static void ApplyTableBorders(ZipArchive archive, string worksheetPath, - List> rows, Dictionary tableStyleBorders) + IList> rows, Dictionary tableStyleBorders) { if (tableStyleBorders.Count == 0) return; @@ -562,7 +563,7 @@ private static void ApplyTableBorders(ZipArchive archive, string worksheetPath, /// then override them, matching Office/LibreOffice precedence for custom tables. /// private static void ApplyTableStyleFormatting(ZipArchive archive, string worksheetPath, - List> rows, IReadOnlyList dxfInfos) + IList> rows, IReadOnlyList dxfInfos) { var relsPath = GetRelationshipPartPath(worksheetPath); var relsEntry = archive.GetEntry(relsPath); @@ -1396,7 +1397,7 @@ private static Dictionary ReadDefinedNameValues(ZipArchive archi /// Applies conditional formatting rules to cell data, overriding cell appearance where rules match. /// private static void ApplyConditionalFormatting( - List> rows, + IList> rows, List<(string Sqref, string Type, string Operator, string Formula, int DxfId)> cfRules, List dxfStyles, Dictionary definedNames) @@ -1458,7 +1459,7 @@ private static void ApplyConditionalFormatting( } } - private static bool EvaluateCondition(List> rows, ExcelCell cell, string type, string op, string formula, + private static bool EvaluateCondition(IList> rows, ExcelCell cell, string type, string op, string formula, int targetRow, int targetCol, int anchorRow, int anchorCol, Dictionary definedNames) { @@ -1527,7 +1528,7 @@ private static bool EvaluateCondition(List> rows, ExcelCell cell return false; } - private static bool? EvaluateConditionalExpression(List> rows, string formula, + private static bool? EvaluateConditionalExpression(IList> rows, string formula, int targetRow, int targetCol, int anchorRow, int anchorCol, Dictionary definedNames) { var trimmed = formula.Trim(); @@ -1588,7 +1589,7 @@ private static bool TryResolveConditionalCellReference(System.Text.RegularExpres return row >= 0 && col >= 0; } - private static bool TryResolveRhsValue(List> rows, string rhs, + private static bool TryResolveRhsValue(IList> rows, string rhs, int targetRow, int targetCol, int anchorRow, int anchorCol, Dictionary definedNames, out string? text, out double? number) { @@ -1631,7 +1632,7 @@ private static bool TryResolveRhsValue(List> rows, string rhs, return true; } - private static string GetCellText(List> rows, int row, int col) + private static string GetCellText(IList> rows, int row, int col) => row >= 0 && row < rows.Count && col >= 0 && col < rows[row].Count ? rows[row][col].Text : string.Empty; @@ -1813,15 +1814,41 @@ private static FontStyleInfo ResolveFontStyle(int styleIndex, List> ReadSheet(ZipArchiveEntry entry, List sharedStrings, Dictionary boldPrefixLengths, + private static IList> ReadSheet(ZipArchiveEntry entry, IList sharedStrings, Dictionary boldPrefixLengths, List fontStyles, List fillColors, List borders, Dictionary numberFormats, List cellXfFontIndices, List cellXfFillIndices, List cellXfNumFmtIds, List cellXfAlignments, List cellXfVerticalAlignments, List cellXfTextRotations, List cellXfBorderIndices, List cellXfWrapTexts, List cellXfIndents, - Dictionary? hyperlinks = null, System.Globalization.CultureInfo? culture = null) + Dictionary? hyperlinks = null, System.Globalization.CultureInfo? culture = null, + LowMemoryWorkbook? lowMemory = null, XDocument? worksheet = null) { + if (lowMemory != null) + { + var storedRows = lowMemory.CreateRows(); + using var metadataStream = entry.Open(); + var metadata = XDocument.Load(metadataStream); + var sheetNamespace = metadata.Root?.GetDefaultNamespace() ?? XNamespace.None; + var columns = metadata.Descendants(sheetNamespace + "cols").FirstOrDefault(); + foreach (var xml in lowMemory.GetRows(entry.FullName)) + { + if (xml.Length == 0) { storedRows.Add([]); continue; } + var rowDocument = new XDocument(new XElement(sheetNamespace + "worksheet", + new XAttribute("xmlns", sheetNamespace.NamespaceName), + columns == null ? null : new XElement(columns), + new XElement(sheetNamespace + "sheetData", XElement.Parse(xml)))); + storedRows.Add(ReadSheet(entry, sharedStrings, boldPrefixLengths, fontStyles, fillColors, borders, + numberFormats, cellXfFontIndices, cellXfFillIndices, cellXfNumFmtIds, cellXfAlignments, + cellXfVerticalAlignments, cellXfTextRotations, cellXfBorderIndices, cellXfWrapTexts, + cellXfIndents, hyperlinks, culture, worksheet: rowDocument)[0]); + } + return storedRows; + } var rows = new List>(); - using var stream = entry.Open(); - var doc = XDocument.Load(stream); + var doc = worksheet; + if (doc == null) + { + using var stream = entry.Open(); + doc = XDocument.Load(stream); + } var ns = doc.Root?.GetDefaultNamespace() ?? XNamespace.None; // Hyperlink cells often rely on workbook defaults for visual style. @@ -1903,6 +1930,8 @@ private static List> ReadSheet(ZipArchiveEntry entry, List= 16384)) + throw new InvalidDataException("Cell column references must be within the XLSX column limit."); // Fill empty cells for gaps, applying row/column default fills if applicable. while (lastColIndex < colIndex) @@ -4744,7 +4773,7 @@ internal sealed record ExcelChartInfo( internal sealed class ExcelSheet { public string Name { get; } - public List> Rows { get; } + public IList> Rows { get; } public List Images { get; } public List Shapes { get; } public List Charts { get; } @@ -4827,7 +4856,7 @@ public float CharUnitsToPointsDefaultScaled(float charUnits) return charUnits * 5.62f * ratio; } - internal ExcelSheet(string name, List> rows, + internal ExcelSheet(string name, IList> rows, List? images = null, Dictionary? columnWidths = null, float defaultColumnWidth = 8.43f, diff --git a/src/MiniPdf/ExcelToPdfConverter.cs b/src/MiniPdf/ExcelToPdfConverter.cs index 1cb1b142..16908424 100644 --- a/src/MiniPdf/ExcelToPdfConverter.cs +++ b/src/MiniPdf/ExcelToPdfConverter.cs @@ -84,6 +84,8 @@ internal sealed class ConversionOptions /// Optional culture used to format numeric and date values. Null uses invariant culture. internal System.Globalization.CultureInfo? Culture { get; set; } + + internal bool LowMemory { get; set; } } /// @@ -107,41 +109,63 @@ internal static PdfDocument Convert(string excelPath, ConversionOptions? options internal static PdfDocument Convert(Stream excelStream, ConversionOptions? options = null) { options ??= new ConversionOptions(); - var sheets = ExcelReader.ReadSheets(excelStream, options.Culture); + using var lowMemory = options.LowMemory ? new LowMemoryWorkbook() : null; + if (lowMemory != null) + { + if (options.MaxRows.HasValue || options.MaxColumns.HasValue) + LowMemoryWorkbook.Unsupported("MaxRows or MaxColumns"); + if (options.FitToHeight is > 0 || options.RowsPerPage.HasValue) + LowMemoryWorkbook.Unsupported("FitToHeight or RowsPerPage"); + lowMemory.Prepare(excelStream); + } + var sheets = ExcelReader.ReadSheets(lowMemory?.Package ?? excelStream, options.Culture, lowMemory); sheets = FilterSheets(sheets, options.Sheets, options.SheetIndexes); sheets = ApplyRenderLimits(sheets, options.MaxRows, options.MaxColumns); sheets = ApplyLayoutOptions(sheets, options); - var doc = new PdfDocument(); + if (options.LowMemory) + foreach (var sheet in sheets) + if (sheet.FitToPage && sheet.FitToHeight > 0) + LowMemoryWorkbook.Unsupported("fit-to-height pagination"); + var doc = new PdfDocument(options.LowMemory); - // Track page ranges per sheet for footer rendering - var sheetPageRanges = new List<(ExcelSheet sheet, int startPage, int endPage)>(); - foreach (var sheet in sheets) + var completed = false; + try { - var startPage = doc.Pages.Count; - RenderSheet(doc, sheet, options, sheets.Count > 1); - var endPage = doc.Pages.Count - 1; - if (startPage <= endPage) - sheetPageRanges.Add((sheet, startPage, endPage)); - } + // Track page ranges per sheet for footer rendering + var sheetPageRanges = new List<(ExcelSheet sheet, int startPage, int endPage)>(); + foreach (var sheet in sheets) + { + var startPage = doc.Pages.Count; + RenderSheet(doc, sheet, options, sheets.Count > 1); + var endPage = doc.Pages.Count - 1; + if (startPage <= endPage) + sheetPageRanges.Add((sheet, startPage, endPage)); + } - // If no sheets found, create at least one empty page - if (doc.Pages.Count == 0) - { - doc.AddPage(options.PageWidth, options.PageHeight); - } + // If no sheets found, create at least one empty page + if (doc.Pages.Count == 0) + { + doc.AddPage(options.PageWidth, options.PageHeight); + } + + // Render print footers on each page + var totalPages = doc.Pages.Count; + foreach (var (sheet, startPage, endPage) in sheetPageRanges) + { + if (string.IsNullOrEmpty(sheet.OddFooter)) continue; + var footerY = sheet.FooterMarginPt > 0 ? sheet.FooterMarginPt : 18f; + var marginL = sheet.MarginLeftPt > 0 ? sheet.MarginLeftPt : options.MarginLeft; + var marginR = sheet.MarginRightPt > 0 ? sheet.MarginRightPt : options.MarginRight; + RenderPageFooter(doc.Pages, sheet.OddFooter, startPage, endPage, totalPages, footerY, marginL, marginR); + } - // Render print footers on each page - var totalPages = doc.Pages.Count; - foreach (var (sheet, startPage, endPage) in sheetPageRanges) + completed = true; + return doc; + } + finally { - if (string.IsNullOrEmpty(sheet.OddFooter)) continue; - var footerY = sheet.FooterMarginPt > 0 ? sheet.FooterMarginPt : 18f; - var marginL = sheet.MarginLeftPt > 0 ? sheet.MarginLeftPt : options.MarginLeft; - var marginR = sheet.MarginRightPt > 0 ? sheet.MarginRightPt : options.MarginRight; - RenderPageFooter(doc.Pages, sheet.OddFooter, startPage, endPage, totalPages, footerY, marginL, marginR); + if (!completed) doc.Dispose(); } - - return doc; } private static List FilterSheets(List sheets, string[]? selectedSheets, int[]? selectedSheetIndexes) @@ -349,7 +373,7 @@ private static int GetLastRowIndex(ExcelSheet sheet) /// Optional conversion settings. internal static void ConvertToFile(string excelPath, string pdfPath, ConversionOptions? options = null) { - var doc = Convert(excelPath, options); + using var doc = Convert(excelPath, options); doc.Save(pdfPath); } @@ -712,6 +736,7 @@ private static void RenderSheet(PdfDocument doc, ExcelSheet sheet, ConversionOpt PageHeight = baseH, IncludeSheetName = options.IncludeSheetName, ScaleCellFonts = scaleCellFonts, + LowMemory = options.LowMemory, }; // Apply print area: limit rows and columns to the defined range @@ -1742,6 +1767,12 @@ float RenderPrintTitleRows() // Render rows foreach (var row in sheet.Rows) { + if (options.LowMemory) + { + // Drawing anchors are unsupported, so do not retain references to completed pages. + rowTopY.Clear(); + rowPage.Clear(); + } EnsurePage(); // Manual row break: force a new page before this row diff --git a/src/MiniPdf/FileBackedList.cs b/src/MiniPdf/FileBackedList.cs new file mode 100644 index 00000000..56b80328 --- /dev/null +++ b/src/MiniPdf/FileBackedList.cs @@ -0,0 +1,309 @@ +using System.Collections; +using System.Text; + +namespace MiniSoftware; + +// Keeps one decoded item in memory. Only the offset index grows with the document. +internal sealed class FileBackedList : IList, IReadOnlyList, IDisposable +{ + private readonly FileStream _stream; + private readonly BinaryReader _reader; + private readonly BinaryWriter _writer; + private readonly Func _read; + private readonly Action _write; + private readonly List _offsets = []; + private readonly bool _writeBack; + private int _cachedIndex = -1; + private T _cachedItem = default!; + + internal FileBackedList(Func read, Action write, bool writeBack = false) + { + _stream = CreateTemporaryStream(); + _reader = new BinaryReader(_stream, Encoding.UTF8, leaveOpen: true); + _writer = new BinaryWriter(_stream, Encoding.UTF8, leaveOpen: true); + _read = read; + _write = write; + _writeBack = writeBack; + } + + internal static FileStream CreateTemporaryStream() + => new(Path.Combine(Path.GetTempPath(), "minipdf-" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 65536, FileOptions.DeleteOnClose); + + public int Count => _offsets.Count; + public bool IsReadOnly => false; + internal string FilePath => _stream.Name; + + public T this[int index] + { + get + { + if (index < 0 || index >= Count) throw new ArgumentOutOfRangeException(nameof(index)); + if (_cachedIndex == index) return _cachedItem; + FlushCachedItem(); + _writer.Flush(); + _stream.Position = _offsets[index]; + _cachedItem = _read(_reader); + _cachedIndex = index; + return _cachedItem; + } + set + { + if (index < 0 || index >= Count) throw new ArgumentOutOfRangeException(nameof(index)); + FlushCachedItem(); + _cachedIndex = index; + _cachedItem = value; + Store(index, value); + } + } + + public void Add(T item) + { + FlushCachedItem(); + _offsets.Add(0); + _cachedIndex = Count - 1; + _cachedItem = item; + Store(_cachedIndex, item); + } + + private void Store(int index, T item) + { + _stream.Position = _stream.Length; + _offsets[index] = _stream.Position; + _write(_writer, item); + _writer.Flush(); + } + + private void FlushCachedItem() + { + if (_writeBack && _cachedIndex >= 0) Store(_cachedIndex, _cachedItem); + } + + public void RemoveAt(int index) + { + if (index != Count - 1 || index < 0) + throw new NotSupportedException("File-backed collections only support removing the last item."); + FlushCachedItem(); + _offsets.RemoveAt(index); + _cachedIndex = -1; + _cachedItem = default!; + } + + public void Clear() + { + _offsets.Clear(); + _cachedIndex = -1; + _cachedItem = default!; + _stream.SetLength(0); + } + + public IEnumerator GetEnumerator() + { + for (var i = 0; i < Count; i++) yield return this[i]; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public int IndexOf(T item) + { + for (var i = 0; i < Count; i++) + if (EqualityComparer.Default.Equals(this[i], item)) return i; + return -1; + } + + public bool Contains(T item) => IndexOf(item) >= 0; + public void CopyTo(T[] array, int arrayIndex) + { + if (array == null) throw new ArgumentNullException(nameof(array)); + if (arrayIndex < 0 || arrayIndex > array.Length) throw new ArgumentOutOfRangeException(nameof(arrayIndex)); + if (array.Length - arrayIndex < Count) throw new ArgumentException("Destination array is too small.", nameof(array)); + foreach (var item in this) array[arrayIndex++] = item; + } + + public void Insert(int index, T item) => throw new NotSupportedException(); + public bool Remove(T item) => throw new NotSupportedException(); + + public void Dispose() + { + try + { + _reader.Dispose(); + _writer.Dispose(); + } + finally + { + _stream.Dispose(); + _cachedItem = default!; + } + } +} + +internal static class LowMemorySerialization +{ + private static void WriteString(BinaryWriter writer, string? value) + { + writer.Write(value != null); + if (value != null) writer.Write(value); + } + + private static string? ReadString(BinaryReader reader) => reader.ReadBoolean() ? reader.ReadString() : null; + private static void WriteFloat(BinaryWriter writer, float? value) + { + writer.Write(value.HasValue); + if (value.HasValue) writer.Write(value.Value); + } + + private static float? ReadFloat(BinaryReader reader) => reader.ReadBoolean() ? reader.ReadSingle() : null; + private static void WriteColor(BinaryWriter writer, PdfColor? color) + { + writer.Write(color.HasValue); + if (color is { } c) { writer.Write(c.R); writer.Write(c.G); writer.Write(c.B); } + } + + private static PdfColor? ReadColor(BinaryReader reader) + => reader.ReadBoolean() ? new PdfColor(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()) : null; + + private static void WriteBorderSide(BinaryWriter writer, BorderSide? side) + { + writer.Write(side != null); + if (side != null) { writer.Write(side.Style); WriteColor(writer, side.Color); } + } + + private static BorderSide? ReadBorderSide(BinaryReader reader) + => reader.ReadBoolean() ? new BorderSide(reader.ReadString(), ReadColor(reader)) : null; + + internal static void WriteRow(BinaryWriter writer, List row) + { + writer.Write(row.Count); + foreach (var cell in row) + { + writer.Write(cell.Text); + WriteColor(writer, cell.Color); + WriteColor(writer, cell.FillColor); + writer.Write(cell.Alignment); + writer.Write(cell.FontSize); + writer.Write(cell.Bold); + writer.Write(cell.Italic); + writer.Write(cell.Underline); + writer.Write(cell.Strikethrough); + writer.Write(cell.Border != null); + if (cell.Border is { } border) + { + WriteBorderSide(writer, border.Left); + WriteBorderSide(writer, border.Right); + WriteBorderSide(writer, border.Top); + WriteBorderSide(writer, border.Bottom); + } + writer.Write(cell.VerticalAlignment); + writer.Write(cell.WrapText); + WriteString(writer, cell.AccountingPrefix); + WriteString(writer, cell.FontName); + writer.Write(cell.Indent); + writer.Write(cell.BoldPrefixLength); + WriteString(writer, cell.Link); + writer.Write(cell.TextRotation); + } + } + + internal static List ReadRow(BinaryReader reader) + { + var count = reader.ReadInt32(); + var row = new List(count); + for (var i = 0; i < count; i++) + { + var text = reader.ReadString(); + var color = ReadColor(reader); + var fill = ReadColor(reader); + var alignment = reader.ReadString(); + var size = reader.ReadSingle(); + var bold = reader.ReadBoolean(); + var italic = reader.ReadBoolean(); + var underline = reader.ReadBoolean(); + var strike = reader.ReadBoolean(); + var border = reader.ReadBoolean() + ? new CellBorderInfo(ReadBorderSide(reader), ReadBorderSide(reader), ReadBorderSide(reader), ReadBorderSide(reader)) + : null; + row.Add(new ExcelCell(text, color, fill, alignment, size, bold, italic, underline, strike, border, + reader.ReadString(), reader.ReadBoolean(), ReadString(reader), ReadString(reader), + reader.ReadInt32(), reader.ReadInt32(), ReadString(reader), reader.ReadInt32())); + } + return row; + } + + internal static void WritePage(BinaryWriter writer, PdfPage page) + { + if (page.ImageBlocks.Count != 0 || page.EllipseBlocks.Count != 0 || page.PolygonBlocks.Count != 0 || + page.PathBlocks.Count != 0 || page.OverlayRects.Count != 0 || page.OverlayLines.Count != 0 || + page.OverlayTexts.Count != 0) + throw new NotSupportedException("LowMemory mode only supports worksheet text, cell fills and borders."); + writer.Write(page.Width); + writer.Write(page.Height); + writer.Write(page.TextBlocks.Count); + foreach (var text in page.TextBlocks) + { + writer.Write(text.Text); + writer.Write(text.X); writer.Write(text.Y); writer.Write(text.FontSize); + WriteColor(writer, text.Color); + writer.Write(text.ClipRect.HasValue); + if (text.ClipRect is { } clip) + { + writer.Write(clip.X); writer.Write(clip.Y); writer.Write(clip.Width); writer.Write(clip.Height); + } + WriteFloat(writer, text.MaxWidth); + writer.Write(text.Bold); writer.Write(text.Italic); writer.Write(text.Underline); + writer.Write(text.CharSpacing); writer.Write(text.WordSpacing); + WriteString(writer, text.PreferredFontName); + WriteFloat(writer, text.UnderlineWidth); + writer.Write(text.Strikethrough); writer.Write(text.Hidden); + } + writer.Write(page.RectBlocks.Count); + foreach (var rect in page.RectBlocks) + { + writer.Write(rect.X); writer.Write(rect.Y); writer.Write(rect.Width); writer.Write(rect.Height); + WriteColor(writer, rect.FillColor); writer.Write(rect.Alpha); + } + writer.Write(page.LineBlocks.Count); + foreach (var line in page.LineBlocks) + { + writer.Write(line.X1); writer.Write(line.Y1); writer.Write(line.X2); writer.Write(line.Y2); + WriteColor(writer, line.Color); writer.Write(line.LineWidth); + writer.Write(line.DashPattern?.Length ?? -1); + if (line.DashPattern != null) + foreach (var dash in line.DashPattern) writer.Write(dash); + } + } + + internal static PdfPage ReadPage(BinaryReader reader) + { + var page = new PdfPage(reader.ReadSingle(), reader.ReadSingle()); + var texts = reader.ReadInt32(); + for (var i = 0; i < texts; i++) + { + var text = reader.ReadString(); + var x = reader.ReadSingle(); var y = reader.ReadSingle(); var size = reader.ReadSingle(); + var color = ReadColor(reader); + (float, float, float, float)? clip = reader.ReadBoolean() + ? (reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle()) : null; + page.AddText(text, x, y, size, color, clip, ReadFloat(reader), + reader.ReadBoolean(), reader.ReadBoolean(), reader.ReadBoolean(), + reader.ReadSingle(), reader.ReadSingle(), ReadString(reader), ReadFloat(reader), + reader.ReadBoolean(), reader.ReadBoolean()); + } + var rectangles = reader.ReadInt32(); + for (var i = 0; i < rectangles; i++) + page.AddRectangle(reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), reader.ReadSingle(), + ReadColor(reader), reader.ReadSingle()); + var lines = reader.ReadInt32(); + for (var i = 0; i < lines; i++) + { + var x1 = reader.ReadSingle(); var y1 = reader.ReadSingle(); + var x2 = reader.ReadSingle(); var y2 = reader.ReadSingle(); + var color = ReadColor(reader); var width = reader.ReadSingle(); + var count = reader.ReadInt32(); + float[]? dashes = count < 0 ? null : new float[count]; + if (dashes != null) + for (var d = 0; d < count; d++) dashes[d] = reader.ReadSingle(); + page.AddLine(x1, y1, x2, y2, color, width, dashes); + } + return page; + } +} diff --git a/src/MiniPdf/LowMemoryWorkbook.cs b/src/MiniPdf/LowMemoryWorkbook.cs new file mode 100644 index 00000000..1fcd38f3 --- /dev/null +++ b/src/MiniPdf/LowMemoryWorkbook.cs @@ -0,0 +1,166 @@ +using System.IO.Compression; +using System.Xml; +using System.Xml.Linq; + +namespace MiniSoftware; + +internal sealed class LowMemoryWorkbook : IDisposable +{ + private readonly List _resources = []; + private readonly Dictionary> _rows = new(StringComparer.Ordinal); + internal FileStream Package { get; } + internal FileBackedList SharedStrings { get; } + internal Dictionary BoldPrefixLengths { get; } = []; + + internal LowMemoryWorkbook() + { + Package = Own(FileBackedList.CreateTemporaryStream()); + var initialized = false; + try + { + SharedStrings = Own(new FileBackedList(reader => reader.ReadString(), (writer, text) => writer.Write(text))); + initialized = true; + } + finally + { + if (!initialized) Package.Dispose(); + } + } + + private T Own(T resource) where T : IDisposable + { + _resources.Add(resource); + return resource; + } + + internal FileBackedList> CreateRows() + => Own(new FileBackedList>(LowMemorySerialization.ReadRow, LowMemorySerialization.WriteRow)); + + internal IEnumerable GetRows(string path) => _rows[path]; + + internal void Prepare(Stream input) + { + using var source = new ZipArchive(input, ZipArchiveMode.Read, leaveOpen: true); + using (var target = new ZipArchive(Package, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var entry in source.Entries) + { + using var entryStream = entry.Open(); + if (entry.FullName == "xl/sharedStrings.xml") + { + ReadSharedStrings(entryStream); + continue; + } + var outputEntry = target.CreateEntry(entry.FullName); + using var output = outputEntry.Open(); + if (entry.FullName.StartsWith("xl/worksheets/", StringComparison.Ordinal) && + entry.FullName.EndsWith(".xml", StringComparison.Ordinal) && + !entry.FullName.Contains("/_rels/")) + { + var rows = Own(new FileBackedList(reader => reader.ReadString(), (writer, text) => writer.Write(text))); + _rows.Add(entry.FullName, rows); + CopyWorksheet(entryStream, output, rows); + } + else if (entry.FullName == "xl/workbook.xml") + { + var workbook = XDocument.Load(entryStream); + foreach (var name in workbook.Descendants().Where(element => element.Name.LocalName == "definedName")) + { + var value = name.Attribute("name")?.Value; + if (value is "_xlnm.Print_Area" or "_xlnm.Print_Titles") + Unsupported(value); + } + workbook.Save(output); + } + else + { + entryStream.CopyTo(output); + } + } + } + Package.Position = 0; + } + + private void ReadSharedStrings(Stream input) + { + using var reader = XmlReader.Create(input, new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit }); + reader.Read(); + while (!reader.EOF) + { + if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "si") + { + var item = (XElement)XNode.ReadFrom(reader); + var (text, boldPrefix) = ExcelReader.ReadSharedString(item); + if (boldPrefix > 0) BoldPrefixLengths[SharedStrings.Count] = boldPrefix; + SharedStrings.Add(text); + } + else reader.Read(); + } + } + + private static void CopyWorksheet(Stream input, Stream output, FileBackedList rows) + { + using var reader = XmlReader.Create(input, new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit }); + using var writer = XmlWriter.Create(output, new XmlWriterSettings { CloseOutput = false }); + reader.Read(); + var lastRow = 0; + while (!reader.EOF) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.LocalName) + { + case "mergeCells": + case "conditionalFormatting": + case "drawing": + case "legacyDrawing": + case "legacyDrawingHF": + case "tableParts": + case "hyperlinks": + case "headerFooter": + case "extLst": + Unsupported(reader.LocalName); + break; + case "printOptions": + if (reader.GetAttribute("verticalCentered") is "1" or "true") + Unsupported("vertical centering"); + break; + case "row": + var row = (XElement)XNode.ReadFrom(reader); + if (row.Attribute("ht") != null || row.Attribute("hidden") is { Value: "1" or "true" } || + row.Attribute("customHeight") is { Value: "1" or "true" }) + Unsupported("custom or hidden row heights"); + if (row.Descendants().Any(element => element.Name.LocalName == "f")) + Unsupported("formulas"); + var rowNumber = row.Attribute("r") == null ? lastRow + 1 : (int)row.Attribute("r")!; + if (rowNumber <= lastRow || rowNumber > 1048576) + throw new InvalidDataException("Worksheet rows must be ordered and within the XLSX row limit."); + while (++lastRow < rowNumber) rows.Add(string.Empty); + // Each row is parsed independently; its original position is kept by the disk index. + row.SetAttributeValue("r", 1); + rows.Add(row.ToString(SaveOptions.DisableFormatting)); + continue; + } + writer.WriteStartElement(reader.Prefix, reader.LocalName, reader.NamespaceURI); + writer.WriteAttributes(reader, defattr: false); + if (reader.IsEmptyElement) writer.WriteEndElement(); + } + else if (reader.NodeType == XmlNodeType.EndElement) writer.WriteFullEndElement(); + else if (reader.NodeType is XmlNodeType.Text or XmlNodeType.SignificantWhitespace or XmlNodeType.Whitespace) + writer.WriteString(reader.Value); + else if (reader.NodeType == XmlNodeType.CDATA) writer.WriteCData(reader.Value); + else if (reader.NodeType == XmlNodeType.Comment) writer.WriteComment(reader.Value); + else if (reader.NodeType == XmlNodeType.ProcessingInstruction) + writer.WriteProcessingInstruction(reader.Name, reader.Value); + reader.Read(); + } + } + + internal static void Unsupported(string feature) + => throw new NotSupportedException($"XLSX LowMemory mode does not support {feature}. Disable LowMemory to use the existing converter."); + + public void Dispose() + { + for (var i = _resources.Count - 1; i >= 0; i--) _resources[i].Dispose(); + } +} diff --git a/src/MiniPdf/MiniPdf.cs b/src/MiniPdf/MiniPdf.cs index c233ecf9..f159ffd7 100644 --- a/src/MiniPdf/MiniPdf.cs +++ b/src/MiniPdf/MiniPdf.cs @@ -19,6 +19,12 @@ public sealed class MiniPdfConversionOptions /// Compress PDF page content streams using FlateDecode. public bool Compress { get; set; } + /// + /// Use disk-backed XLSX rows and pages to reduce memory usage. Defaults to false. + /// Unsupported worksheet features throw NotSupportedException. Requires writable temporary storage. + /// + public bool LowMemory { get; set; } + /// Optional DOCX page size override in PDF points. public MiniPdfPageSize? PageSize { get; set; } @@ -215,7 +221,7 @@ public static void ClearRegisteredFonts() public static void ConvertToPdf(string inputPath, string outputPath, MiniPdfConversionOptions? options = null) { options = PrepareConversionOptions(options); - var document = ConvertPathToDocument(inputPath, options); + using var document = ConvertPathToDocument(inputPath, options); document.Save(outputPath, CreatePdfSaveOptions(options)); } @@ -228,7 +234,7 @@ public static void ConvertToPdf(string inputPath, string outputPath, MiniPdfConv public static byte[] ConvertToPdf(string inputPath, MiniPdfConversionOptions? options = null) { options = PrepareConversionOptions(options); - var document = ConvertPathToDocument(inputPath, options); + using var document = ConvertPathToDocument(inputPath, options); return document.ToArray(CreatePdfSaveOptions(options)); } @@ -266,7 +272,7 @@ public static void ConvertToPdf(Stream inputStream, Stream outputStream, MiniPdf throw new ArgumentException("Output stream must be writable.", nameof(outputStream)); options = PrepareConversionOptions(options); - var document = ConvertStreamToDocument(inputStream, options); + using var document = ConvertStreamToDocument(inputStream, options); document.Save(outputStream, CreatePdfSaveOptions(options)); } @@ -313,15 +319,19 @@ private static PdfDocument ConvertStreamToDocument(Stream inputStream, MiniPdfCo } else { - var ms = new MemoryStream(); - inputStream.CopyTo(ms); - ms.Position = 0; - seekable = ms; + seekable = options.LowMemory + ? FileBackedList.CreateTemporaryStream() + : new MemoryStream(); ownsSeekable = true; } try { + if (ownsSeekable) + { + inputStream.CopyTo(seekable); + seekable.Position = 0; + } var startPosition = seekable.Position; var format = DetectOfficeFormat(seekable); seekable.Position = startPosition; @@ -391,7 +401,7 @@ private static void ThrowIfSheetsSpecifiedForNonXlsx(string[]? sheets, int[]? sh private static void ThrowIfXlsxOnlyOptionsSpecifiedForNonXlsx(MiniPdfConversionOptions options) { ThrowIfSheetsSpecifiedForNonXlsx(options.Sheets, options.SheetIndexes); - if (options.MaxRows.HasValue || options.MaxColumns.HasValue || + if (options.LowMemory || options.MaxRows.HasValue || options.MaxColumns.HasValue || options.Landscape.HasValue || options.FitToPage.HasValue || options.FitToWidth.HasValue || options.FitToHeight.HasValue || options.PrintScale.HasValue || options.RowsPerPage.HasValue || @@ -461,6 +471,7 @@ private static ExcelToPdfConverter.ConversionOptions CreateExcelOptions(MiniPdfC PrintScale = options.PrintScale, RowsPerPage = options.RowsPerPage, Culture = options.Culture, + LowMemory = options.LowMemory, }; private static PdfSaveOptions? CreatePdfSaveOptions(MiniPdfConversionOptions options) diff --git a/src/MiniPdf/PdfDocument.cs b/src/MiniPdf/PdfDocument.cs index 11c48d5a..94a8fd1f 100644 --- a/src/MiniPdf/PdfDocument.cs +++ b/src/MiniPdf/PdfDocument.cs @@ -3,14 +3,24 @@ namespace MiniSoftware; /// /// Represents a PDF document that can contain pages with text content. /// -internal sealed class PdfDocument +internal sealed class PdfDocument : IDisposable { private readonly List _pages = []; + private readonly FileBackedList? _storedPages; + + internal PdfDocument(bool lowMemory = false) + { + if (lowMemory) + _storedPages = new FileBackedList(LowMemorySerialization.ReadPage, + LowMemorySerialization.WritePage, writeBack: true); + } + + internal bool LowMemory => _storedPages != null; /// /// Gets the pages in this document. /// - public IReadOnlyList Pages => _pages; + public IReadOnlyList Pages => _storedPages != null ? _storedPages : _pages; /// /// Preferred CJK (Chinese/Japanese/Korean) font name from the source document. @@ -27,7 +37,8 @@ internal sealed class PdfDocument public PdfPage AddPage(float width = 612, float height = 792) { var page = new PdfPage(width, height); - _pages.Add(page); + if (_storedPages != null) _storedPages.Add(page); + else _pages.Add(page); return page; } @@ -36,6 +47,8 @@ public PdfPage AddPage(float width = 612, float height = 792) /// internal void RemoveEmptyPages() { + if (_storedPages != null) + throw new NotSupportedException("Removing arbitrary pages is not supported in LowMemory mode."); _pages.RemoveAll(p => p.TextBlocks.Count == 0 && p.ImageBlocks.Count == 0 && p.RectBlocks.Count == 0 && p.LineBlocks.Count == 0); } @@ -44,6 +57,11 @@ internal void RemoveEmptyPages() /// internal void RemoveLastPage() { + if (_storedPages != null) + { + if (_storedPages.Count > 0) _storedPages.RemoveAt(_storedPages.Count - 1); + return; + } if (_pages.Count > 0) _pages.RemoveAt(_pages.Count - 1); } @@ -96,6 +114,8 @@ internal byte[] ToArray(PdfSaveOptions? options) Save(ms, options); return ms.ToArray(); } + + public void Dispose() => _storedPages?.Dispose(); } internal sealed class PdfSaveOptions diff --git a/src/MiniPdf/PdfWriter.cs b/src/MiniPdf/PdfWriter.cs index 7a479654..c59f4ffb 100644 --- a/src/MiniPdf/PdfWriter.cs +++ b/src/MiniPdf/PdfWriter.cs @@ -491,9 +491,10 @@ void LoadFontFile(string path) } // Pre-build content streams - var contentStreams = new List(pageCount); - for (var i = 0; i < pageCount; i++) - contentStreams.Add(Compat.Latin1.GetBytes(BuildContentStream(pages[i], embeddedFonts.Count > 0, cpToFontSlot, embeddedFonts, fontNameToSlot))); + var contentStreams = new List(document.LowMemory ? 0 : pageCount); + if (!document.LowMemory) + for (var i = 0; i < pageCount; i++) + contentStreams.Add(Compat.Latin1.GetBytes(BuildContentStream(pages[i], embeddedFonts.Count > 0, cpToFontSlot, embeddedFonts, fontNameToSlot))); // Allocate object numbers. // 1 = Catalog, 2 = Pages, 3 = Font F1 (Helvetica/WinAnsi), 4 = Font F1B (Helvetica-Bold/WinAnsi) @@ -640,7 +641,9 @@ void LoadFontFile(string path) var page = pages[i]; // Content stream - var content = contentStreams[i]; + var content = document.LowMemory + ? Compat.Latin1.GetBytes(BuildContentStream(page, embeddedFonts.Count > 0, cpToFontSlot, embeddedFonts, fontNameToSlot)) + : contentStreams[i]; var encodedContent = _options.CompressContentStreams ? CompressToZlib(content) : content; var filter = _options.CompressContentStreams ? " /Filter /FlateDecode" : ""; _objectOffsets[contentObjNums[i]] = Position; diff --git a/tests/MiniPdf.Tests/LowMemoryXlsxTests.cs b/tests/MiniPdf.Tests/LowMemoryXlsxTests.cs new file mode 100644 index 00000000..8aa5f909 --- /dev/null +++ b/tests/MiniPdf.Tests/LowMemoryXlsxTests.cs @@ -0,0 +1,390 @@ +using System.IO.Compression; +using System.Text; +using System.Xml; + +namespace MiniSoftware.Tests; + +public class LowMemoryXlsxTests +{ + public static IEnumerable SupportedClassicFixtures() + { + var directory = GetCorpusDirectory(); + var numbers = Enumerable.Range(1, 60).Except([35, 36, 38]) + .Concat(Enumerable.Range(121, 7)) + .Concat(Enumerable.Range(131, 6)) + .Concat([139]) + .Concat(Enumerable.Range(143, 6)) + .Concat(Enumerable.Range(151, 18)) + .Concat(Enumerable.Range(170, 11)) + .Concat([182, 184]); + foreach (var number in numbers) + yield return [Directory.GetFiles(directory, $"classic{number:D2}_*.xlsx").Single()]; + } + + [ClassicCorpusTheory] + [MemberData(nameof(SupportedClassicFixtures))] + public void LowMemory_RealClassicFixture_ProducesIdenticalPdf(string path) + { + using var input = File.OpenRead(path); + var expected = MiniPdf.ConvertToPdf(input); + input.Position = 0; + Assert.Equal(expected, MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + private static string GetCorpusDirectory() + { + var root = new DirectoryInfo(AppContext.BaseDirectory); + while (root != null && !File.Exists(Path.Combine(root.FullName, "MiniPdf.sln"))) root = root.Parent; + if (root == null) throw new DirectoryNotFoundException("MiniPdf repository root was not found."); + return Path.Combine(root.FullName, "tests", "MiniPdf.Scripts", "output"); + } + + public sealed class ClassicCorpusTheoryAttribute : TheoryAttribute + { + public ClassicCorpusTheoryAttribute() + { + if (!File.Exists(Path.Combine(GetCorpusDirectory(), "classic01_basic_table_with_headers.xlsx"))) + Skip = "Generate the classic XLSX corpus with tests/MiniPdf.Scripts/generate_classic_xlsx.py."; + } + } + + [Theory] + [InlineData(0, false, false)] + [InlineData(1, false, false)] + [InlineData(200, false, false)] + [InlineData(1500, true, false)] + [InlineData(80, false, true)] + [InlineData(80, true, true)] + public void LowMemory_ProducesByteIdenticalPdf(int rows, bool compress, bool wide) + { + using var input = CreateWorkbook(rows, wide); + var expected = MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { Compress = compress }); + input.Position = 0; + var actual = MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true, Compress = compress }); + Assert.Equal(expected, actual); + } + + [Fact] + public void LowMemory_NonSeekableStreams_ProduceIdenticalPdfAndStayOpen() + { + using var workbook = CreateWorkbook(100); + var expected = MiniPdf.ConvertToPdf(workbook); + workbook.Position = 0; + using var input = new NonSeekableStream(workbook); + using var destination = new MemoryStream(); + using var output = new NonSeekableStream(destination); + MiniPdf.ConvertToPdf(input, output, new MiniPdfConversionOptions { LowMemory = true }); + Assert.Equal(expected, destination.ToArray()); + Assert.True(input.CanRead); + Assert.True(output.CanWrite); + } + + [Fact] + public void LowMemory_FileOverloads_ProduceIdenticalPdf() + { + var inputPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".xlsx"); + var outputPath = Path.ChangeExtension(inputPath, ".pdf"); + try + { + using (var workbook = CreateWorkbook(150)) + using (var file = File.Create(inputPath)) + workbook.CopyTo(file); + var expected = MiniPdf.ConvertToPdf(inputPath); + Assert.Equal(expected, MiniPdf.ConvertToPdf(inputPath, new MiniPdfConversionOptions { LowMemory = true })); + MiniPdf.ConvertToPdf(inputPath, outputPath, new MiniPdfConversionOptions { LowMemory = true }); + Assert.Equal(expected, File.ReadAllBytes(outputPath)); + ExcelToPdfConverter.ConvertToFile(inputPath, outputPath, + new ExcelToPdfConverter.ConversionOptions { LowMemory = true }); + Assert.Equal(expected, File.ReadAllBytes(outputPath)); + } + finally + { + File.Delete(inputPath); + File.Delete(outputPath); + } + } + + [Fact] + public void LowMemory_LayoutOverridesAndCulture_PreserveOutput() + { + using var input = CreateWorkbook(200); + var options = new MiniPdfConversionOptions + { + Landscape = true, + FitToWidth = 1, + FitToHeight = 0, + PrintScale = 80, + Culture = System.Globalization.CultureInfo.GetCultureInfo("fr-FR"), + Sheets = ["Data"], + }; + var expected = MiniPdf.ConvertToPdf(input, options); + input.Position = 0; + options.LowMemory = true; + Assert.Equal(expected, MiniPdf.ConvertToPdf(input, options)); + } + + [Theory] + [InlineData("", "mergeCells")] + [InlineData("", "conditionalFormatting")] + [InlineData("", "drawing")] + [InlineData("", "legacyDrawing")] + [InlineData("", "tableParts")] + [InlineData("", "hyperlinks")] + [InlineData("", "headerFooter")] + [InlineData("", "vertical centering")] + [InlineData("", "extLst")] + public void LowMemory_UnsupportedFeatures_ThrowBeforeWritingOutput(string feature, string message) + { + using var input = CreateWorkbook(1, extraWorksheetXml: feature); + using var output = new MemoryStream(); + var exception = Assert.Throws(() => + MiniPdf.ConvertToPdf(input, output, new MiniPdfConversionOptions { LowMemory = true })); + Assert.Contains(message, exception.Message); + Assert.Equal(0, output.Length); + } + + [Theory] + [InlineData("1", "row heights")] + [InlineData("", "row heights")] + [InlineData("1", "row heights")] + [InlineData("1+12", "formulas")] + public void LowMemory_UnsupportedRows_Throw(string row, string message) + { + using var input = CreateWorkbook(0, customRows: row); + var exception = Assert.Throws(() => + MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + Assert.Contains(message, exception.Message); + } + + [Theory] + [InlineData("")] + [InlineData("")] + [InlineData("1")] + public void LowMemory_InvalidRowOrderOrLimit_Throws(string rows) + { + using var input = CreateWorkbook(0, customRows: rows); + Assert.Throws(() => + MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + [Fact] + public void LowMemory_SparseRowsColumnsAndTrailingEmptyRows_PreserveOutput() + { + using var input = CreateWorkbook(0, customRows: + """ + First + Last + + """); + var expected = MiniPdf.ConvertToPdf(input); + input.Position = 0; + Assert.Equal(expected, MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + [Fact] + public void LowMemory_UnsupportedOptions_Throw() + { + var options = new[] + { + new MiniPdfConversionOptions { LowMemory = true, MaxRows = 10 }, + new MiniPdfConversionOptions { LowMemory = true, MaxColumns = 2 }, + new MiniPdfConversionOptions { LowMemory = true, FitToHeight = 1 }, + new MiniPdfConversionOptions { LowMemory = true, RowsPerPage = 40 }, + }; + foreach (var option in options) + { + using var input = CreateWorkbook(1); + Assert.Throws(() => MiniPdf.ConvertToPdf(input, option)); + } + } + + [Theory] + [InlineData("_xlnm.Print_Area")] + [InlineData("_xlnm.Print_Titles")] + public void LowMemory_PrintDefinitions_Throw(string name) + { + using var input = CreateWorkbook(1); + using (var archive = new ZipArchive(input, ZipArchiveMode.Update, leaveOpen: true)) + { + archive.GetEntry("xl/workbook.xml")!.Delete(); + WriteEntry(archive, "xl/workbook.xml", + $"""Data!$A$1:$B$2"""); + } + input.Position = 0; + Assert.Throws(() => + MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + [Fact] + public void LowMemory_WorksheetFitToHeight_Throws() + { + using var input = CreateWorkbook(1, fitToHeight: 1, + extraWorksheetXml: ""); + Assert.Throws(() => + MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + [Fact] + public void LowMemory_NonXlsxInput_Throws() + { + using var input = new MemoryStream(); + using (var zip = new ZipArchive(input, ZipArchiveMode.Create, leaveOpen: true)) + WriteEntry(zip, "word/document.xml", ""); + input.Position = 0; + Assert.Throws(() => + MiniPdf.ConvertToPdf(input, new MiniPdfConversionOptions { LowMemory = true })); + } + + [Fact] + public void FileBackedPages_RoundTripAndPersistChanges() + { + using var memory = new PdfDocument(); + using var disk = new PdfDocument(lowMemory: true); + foreach (var document in new[] { memory, disk }) + { + for (var i = 0; i < 4; i++) + { + var page = document.AddPage(); + page.AddText("Text " + i, 20, 50, clipRect: (1, 2, 3, 4), maxWidth: 80, + bold: true, italic: true, underline: true, strikethrough: true, hidden: i == 3, + charSpacing: 0.5f, wordSpacing: 1.5f, underlineWidth: 23); + page.AddRectangle(1, 2, 3, 4, PdfColor.Red, 0.5f); + page.AddLine(10, 20, 30, 40, PdfColor.Blue, 2, [2, 3]); + } + document.Pages[0].AddText("Added later", 10, 20); + } + Assert.Equal(memory.ToArray(), disk.ToArray()); + } + + [Fact] + public void TemporaryStorage_IsDeletedOnDispose() + { + string path; + using (var stream = FileBackedList.CreateTemporaryStream()) + { + path = stream.Name; + Assert.True(File.Exists(path)); + } + Assert.False(File.Exists(path)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void LowMemoryWorkbook_DeletesOwnedFilesOnSuccessAndFailure(bool fail) + { + var paths = new List(); + using (var input = CreateWorkbook(10, extraWorksheetXml: fail ? "" : "")) + using (var storage = new LowMemoryWorkbook()) + { + paths.Add(storage.Package.Name); + paths.Add(storage.SharedStrings.FilePath); + if (fail) Assert.Throws(() => storage.Prepare(input)); + else + { + storage.Prepare(input); + var sheets = ExcelReader.ReadSheets(storage.Package, lowMemory: storage); + paths.Add(Assert.IsType>>(Assert.Single(sheets).Rows).FilePath); + } + paths.Add(Assert.IsType>(storage.GetRows("xl/worksheets/sheet1.xml")).FilePath); + Assert.All(paths, path => Assert.True(File.Exists(path))); + } + Assert.All(paths, path => Assert.False(File.Exists(path))); + } + + [Fact] + public void LowMemory_OutputFailure_Propagates() + { + using var input = CreateWorkbook(100); + using var output = new FailingOutput(); + var exception = Assert.Throws(() => + MiniPdf.ConvertToPdf(input, output, new MiniPdfConversionOptions { LowMemory = true })); + Assert.Equal("Output failure", exception.Message); + } + + internal static MemoryStream CreateWorkbook(int rows, bool wide = false, string extraWorksheetXml = "", + string? customRows = null, int fitToHeight = 0) + { + const string ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + var result = new MemoryStream(); + using (var zip = new ZipArchive(result, ZipArchiveMode.Create, leaveOpen: true)) + { + WriteEntry(zip, "xl/workbook.xml", + $""""""); + WriteEntry(zip, "xl/sharedStrings.xml", + $"""Header value中文"""); + WriteEntry(zip, "xl/styles.xml", + $""" + + + + + + + """); + var entry = zip.CreateEntry("xl/worksheets/sheet1.xml"); + using var stream = entry.Open(); + using var writer = XmlWriter.Create(stream); + writer.WriteStartElement("worksheet", ns); + writer.WriteRaw($""); + writer.WriteStartElement("sheetData", ns); + if (customRows != null) writer.WriteRaw(customRows); + else + { + for (var i = 1; i <= rows; i++) + { + writer.WriteStartElement("row", ns); + writer.WriteAttributeString("r", i.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.WriteRaw($"01{i}.25"); + writer.WriteStartElement("c", ns); + writer.WriteAttributeString("r", $"D{i}"); + writer.WriteAttributeString("t", "inlineStr"); + writer.WriteStartElement("is", ns); + writer.WriteElementString("t", ns, $"Row {i} & value"); + writer.WriteEndElement(); + writer.WriteEndElement(); + writer.WriteEndElement(); + } + } + writer.WriteEndElement(); + writer.WriteRaw($"" + extraWorksheetXml); + writer.WriteEndElement(); + } + result.Position = 0; + return result; + } + + private static void WriteEntry(ZipArchive zip, string path, string text) + { + using var writer = new StreamWriter(zip.CreateEntry(path).Open(), Encoding.UTF8); + writer.Write(text); + } + + private sealed class NonSeekableStream(Stream inner) : Stream + { + public override bool CanRead => inner.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => inner.CanWrite; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + } + + private sealed class FailingOutput : Stream + { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override void Write(byte[] buffer, int offset, int count) => throw new IOException("Output failure"); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + } +}