From 0a8c89847d09d7b9d217910bca4803c57a44214e Mon Sep 17 00:00:00 2001 From: David Liebeherr Date: Sun, 25 Oct 2026 22:38:04 +0100 Subject: [PATCH 1/2] docs: specify the performance benchmark suite OpenSpec change add-benchmarks: proposal, design, delta spec and tasks. Refs: #53 --- .../changes/add-benchmarks/.openspec.yaml | 2 + openspec/changes/add-benchmarks/design.md | 399 ++++++++++++++++++ openspec/changes/add-benchmarks/proposal.md | 71 ++++ .../specs/performance-benchmarks/spec.md | 241 +++++++++++ openspec/changes/add-benchmarks/tasks.md | 62 +++ 5 files changed, 775 insertions(+) create mode 100644 openspec/changes/add-benchmarks/.openspec.yaml create mode 100644 openspec/changes/add-benchmarks/design.md create mode 100644 openspec/changes/add-benchmarks/proposal.md create mode 100644 openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md create mode 100644 openspec/changes/add-benchmarks/tasks.md diff --git a/openspec/changes/add-benchmarks/.openspec.yaml b/openspec/changes/add-benchmarks/.openspec.yaml new file mode 100644 index 0000000..1f5d280 --- /dev/null +++ b/openspec/changes/add-benchmarks/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-24 diff --git a/openspec/changes/add-benchmarks/design.md b/openspec/changes/add-benchmarks/design.md new file mode 100644 index 0000000..91baad5 --- /dev/null +++ b/openspec/changes/add-benchmarks/design.md @@ -0,0 +1,399 @@ +# Design + +## Context + +See proposal.md for the motivation and specs/performance-benchmarks/spec.md for the requirements. + +The code paths being measured: + +``` + Code: Localizer.Get(key, args, rm?) + +-- rm given? --> rm.GetString(key, culture) + | no --> loop over registered managers, first hit wins + +-- MessageFormatter.Format(value, args, culture) (runs even without arguments) + default: string.Format | ICU: Dictionary + Jeffijoe FormatMessage + + XAML: ProvideValue --> MultiBinding per element + [0] RefreshTriggeringRevision of Localizer.Current [1] key [2..] Argument values + LocalizedMessageConverter --> MessageArguments.FromTuples --> Localizer.Get + Culture switch --> PropertyChanged --> every live binding calls Get again +``` + +Constraints from the repository: + +- Projects under `tests/` inherit xUnit, coverlet and the test SDK from `tests/Directory.Build.props`, which a + BenchmarkDotNet console app must not carry. Every project still has to pass the repository's analyzers and + style gate (`scripts/tidy-code.ps1`), which the root `Directory.Build.props` imports. +- WPF builds only on Windows; `ResXLocalization.NonWindows.slnf` is what Linux builds use. +- Samples reference the source generator as an analyzer `ProjectReference`; the resx wiring comes from the + root `Directory.Build.targets`. +- GitHub Pages already deploys the DocFX site through `actions/deploy-pages` on pushes to `main`, built with + `--warningsAsErrors`, so a classic `gh-pages` branch would not be served. +- Every third-party action is pinned by commit SHA; Dependabot keeps them current. + +## Goals / Non-Goals + +**Goals:** +- Numbers that answer "will this make my app slow" and "how does it scale" with as little benchmark code as + possible. +- A regression check that gives the same result every time it runs on the same code, even though the speed + of GitHub's shared CI machines varies from run to run. +- Only existing tools: BenchmarkDotNet for measuring, `benchmark-action/github-action-benchmark` for history, + comparison, alerts and charts. + +**Non-Goals:** +- Native AOT benchmarks. The code these benchmarks measure does the same work under AOT; the existing AOT gate covers correctness. + `MessageArguments.FromObject` uses reflection and would be the one to add if AOT numbers are ever wanted. +- .NET 8 benchmarks, multiple operating systems, rendering or layout cost, startup and first-lookup cost + (satellite assembly loading), the source generator's build time. +- Comparisons with other localization libraries; they can be added later. +- Optimizing anything the benchmarks reveal; that is follow-up work. +- Running `main` and the pull request one after the other on the same machine to compare them. + +## Decisions + +### Three projects under a new `benchmarks/` folder, one per suite + +``` + benchmarks/ + Directory.Build.props imports root props, NoWarn, compiles Shared/*.cs + Shared/ [Config], self-check helper (compiled into all three) + ResXLocalization.Core.Benchmarks Core, MessageFormat $(CurrentTargetFramework) + ResXLocalization.Avalonia.Benchmarks Avalonia headless $(CurrentTargetFramework) + ResXLocalization.WPF.Benchmarks WPF $(CurrentWindowsTargetFramework) +``` + +The projects mirror the packages and the test suites. The Core suite answers "did the engine get slower" and +"what does one lookup cost"; the UI suites answer "what does a localized element cost and how does it scale". +Core needs its own suite because a lookup is a small share of a UI benchmark's time and allocations: a +lookup that doubles in cost moves a UI row by a few percent, below both CI thresholds. + +- MessageFormat stays in the Core suite: it is one class, and it measures Core lookups with the ICU + formatter installed. +- `benchmarks/Directory.Build.props` imports the root props, like `tests/` does, adds the `NoWarn` for + `CS1574;CS1591` that samples and tests carry for generated keys, and compiles `benchmarks/Shared/*.cs` into + every benchmark project, so the configuration and the self-check exist once. +- All three join `ResXLocalization.slnx` under `/04_benchmarks/`; Core and Avalonia also join the + non-Windows filter, WPF does not, as in `src/`. They are built by the existing build jobs, so a broken + benchmark fails the build, but they are not test projects and `dotnet test` ignores them. +- The UI suites have one row per extension form (typed key, `Argument`, enum), not every key type and + argument form: those are the Core suite's. + +Alternatives: Core with Avalonia and WPF separate (splits by "builds on Linux", which says nothing about what +a project measures); no Core suite (Core regressions disappear in the UI numbers, and the code-only features - +`MessageArguments`, search-all - are not reached through XAML); a fourth +project for MessageFormat (a project for one class); placing them under `tests/` (inherits the test stack). + +### Resource data + +- **Core**: `Resources/BenchmarkStrings.resx` with about 10 keys (a plain text, texts with positional and + named arguments, one ICU message with two named arguments, the enum values), a `.de` satellite that translates + them, and no `de-DE` satellite. Every benchmark looks up in `de-DE`, which is how most apps ship: a + specific culture with a satellite for its language. The fallback walk `de-DE -> de` is `ResourceManager`'s + and costs the baseline and the library the same; it matters for search-all, where every manager without + the key is walked through the whole chain (`de-DE -> de -> neutral`) before the next one is tried, and + that walk is the library's cost. Search-all benchmarks register 0, 4 or 19 + managers without the key before `BenchmarkStrings`, for 1, 5 or 20 managers in all. They come from one + `Filler.resx` with a single other key: the benchmark creates 19 `ResourceManager` instances over its base + name. The library tells managers apart by instance, and each instance has its own resource-set cache, so + every miss is a real lookup with the full fallback walk, the same work as 19 different files. +- **Avalonia and WPF**: each its own `Resources/BenchmarkStrings.resx` with 10 keys, including an argument + key and the enum keys, with a `.de` satellite. Each 10-element view uses all 10 keys once, so a view has + different texts, as a real one does. + +The key count does not change the numbers: `ResourceManager` keeps a loaded resource set in a dictionary, so +one lookup costs the same with 10 keys or 1,000. Loading the set is a one-time startup cost and is not +measured. + +Every `BenchmarkStrings.resx` has a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` +emits: a public static property per key that calls `ResourceManager.GetString(name, resourceCulture)`. It is +the baseline (see "Baselines"). Typed keys come from the source generator, which handles a resx with a +designer, as in the samples. + +All resx, designer and view files are small enough to write by hand and are committed, like the samples' +files. The samples hand-write their designer files because `PublicResXFileCodeGenerator` runs only inside +Visual Studio. The data is fixed, so nothing needs to regenerate it. + +Alternatives: a script that generates large data sets (1,000 keys, 100-element views), which would run once +and then go out of date, and whose size does not change the results; a build-time generator, which would be +new tooling for a fixed data set. + +### BenchmarkDotNet configuration + +- One `[Config]` shared by all classes: `MemoryDiagnoser`, the JSON (full) and GitHub-markdown and HTML + exporters, and one short job used locally and in CI: + - **3 warmup and 3 measured iterations of 100 ms each**, instead of BenchmarkDotNet's 500 ms and its + automatic choice of more iterations. Fewer, shorter iterations make the mean time vary more between + runs, which is acceptable because time only raises an alert at 150%. Allocations, which fail the check, + do not change with the number of iterations. + - **The warmups stay**: .NET 10 allocates some objects on the stack, but only in fully optimized code. A + benchmark measured before the JIT has optimized it would count allocations a running app does not have + (spec: "Measurements run on optimized code"). + - **In-process toolchain** (`InProcessEmitToolchain`): every benchmark runs in the BenchmarkDotNet host + process instead of a separately built and started process. That removes a build and a process start per + benchmark; the cost is that benchmarks share one process and its GC state, which does not change these + numbers noticeably. The host's `Main` is `[STAThread]` in the WPF project, so the WPF benchmarks run on + an STA thread. +- **Results go to `artifacts/benchmarks//`**: the shared config sets BenchmarkDotNet's + `ArtifactsPath` there, resolved from the repository root, instead of its default `BenchmarkDotNet.Artifacts` + folder in the working directory. `artifacts/` is where the repository puts everything it generates on + purpose: `.gitignore` already excludes it and `scripts/clean-build-artifacts.ps1` already deletes it, so + neither needs a change. +- Each class marks its library-free variant `[Baseline = true]` (see "Baselines"), so the Ratio column is the + overhead. +- `[Params]` for manager count (1, 5, 20) and element count (100, 10,000). Two element counts are enough to + show whether the cost per element stays the same as N grows. +- Each class's `[GlobalSetup]` runs every benchmark method of the class once and throws when the text is not + the expected one, which makes BenchmarkDotNet stop the class with an error (spec: "Benchmarks verify their + own results"). +- `Program.Main` is `BenchmarkSwitcher.FromAssembly(...).Run(args)`, so local runs can filter with + `--filter`. + +### Baselines + +Every baseline is the code an application writes without the library; no other localization library is +benchmarked (a comparison can be added later as extra rows, outside the allocation gate). + +| Benchmark | `[Baseline = true]` | +| --- | --- | +| Code lookup (typed, scoped, search-all, enum) | `BenchmarkStrings.Greeting` (designer property) | +| Code lookup with arguments | `string.Format(culture, BenchmarkStrings.PeopleInvited, count)` | +| ICU MessageFormat | `messageFormatter.FormatMessage(BenchmarkStrings.FilesDeleted, values, culture)` | +| Extension forms (100 elements) | the same 10-element views with `{x:Static}` | +| Opening a view with N elements | the same views with `{x:Static}` | +| Culture switch with N elements | the same views with `{Binding [Key], Source={x:Static local:TranslationSource.Instance}}` | + +- **Culture**: the designer class reads `CultureInfo.CurrentUICulture` while its `Culture` property is unset, + so each `[GlobalSetup]` sets `CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture. +- **ICU**: the baseline uses one `new MessageFormatter()` with the library's defaults, which is what + `UseMessageFormat()` creates, so pattern caching is the same on both sides. The ratio is the cost of + `ResXLocalization.MessageFormat` itself: building the argument dictionary, the culture substitution and the + call through the localizer. What ICU costs compared with `string.Format` is the MessageFormat library's + property and is not measured. One message is enough: plural, select and the other ICU features run inside + the MessageFormat library and cost the baseline the same. The adapter's overhead depends only on the + number of arguments and whether they are named, so the message has two named arguments, which run both + of the adapter's loops (positions and names). +- **`TranslationSource`**: the usual hand-written way to switch languages live, about 20 lines per UI + framework, in each UI project. A singleton implementing `INotifyPropertyChanged` with + `this[string key] => BenchmarkStrings.ResourceManager.GetString(key, this.Culture)` and a `Culture` setter + that raises `PropertyChanged` for the indexer (`"Item[]"`). Elements bind `[key]` on it with the + framework's normal binding. Its culture-switch row sets `TranslationSource.Instance.Culture`; the library's + row sets `Localizer.Current.CurrentCulture`. + +Alternatives: raw `ResourceManager.GetString` (the same cost as the designer property, but code consumers do +not recognize); Microsoft.Extensions.Localization in the Core suite (rarely used in XAML applications, so it +answers no question of this suite); WPFLocalizeExtension in the WPF suite (left for later: new releases of +a third-party project to keep up with, and no equivalent for Avalonia). + +### UI benchmarks use compiled XAML views + +Every UI row, baselines included, is a view written in XAML and compiled by the build, as in a consumer +app: + +``` + Consumer app Benchmark + +-----------------------------+ +---------------------------------------+ + | MainView.axaml / .xaml | | LocalizedView.axaml / .xaml | + | | == | "{l:Localize ...}"/> | + | compiled by the build | | compiled by the build (same tooling) | + | new MainView() -> window | | new LocalizedView() -> window | + +-----------------------------+ +---------------------------------------+ +``` + +- **Avalonia** compiles `.axaml` into IL, so `new LocalizedView()` runs the generated code that creates the + extension, calls `ProvideValue` with the real XAML context and applies the binding. +- **WPF** compiles `.xaml` to BAML, and `InitializeComponent` reads it at run time with WPF's XAML reader and + its service provider. Reading BAML is a cost consumers pay too; the baseline views pay it as well, so the + ratio still shows what `{l:Localize}` adds. +- **Views per UI project**: five 10-element views, written by hand, each using one form for all its + elements: + + ``` + {x:Static} baseline of both benchmarks + {l:Localize} forms and scaling + {l:Localize} + 2 forms only (arguments bound to the view's DataContext) + {l:LocalizeEnum} forms only + TranslationSource scaling only (baseline of the culture switch) + ``` + +- **Forms**: one row per view, each opening 10 instances (100 elements) of it in a window. 100 elements + spread the fixed cost of a view and a window, so a row's ratio to `{x:Static}` is what that form costs per + element in a realistic view. A benchmark with a single element is not used: no app opens one, and its + number would mostly be that fixed cost. +- **Scaling**: plain `{l:Localize}` only. The benchmark puts N / 10 instances of a 10-element view into a + `StackPanel` that is a window's content: 10 or 1,000 views for N = 100 or 10,000, like a + screen built from user controls. One form is enough because what can grow faster than the element count + is shared by all forms: every element subscribes to `Localizer.Current` through its refresh-trigger + binding, and a culture switch updates every subscribed binding. What `` and + `{l:LocalizeEnum}` add - more bindings, building the enum key - is per element and independent of the + other elements, so it is N times the constant the forms benchmark measures. A view with mixed forms costs + about the sum of its elements' per-form costs. +- **What is measured**: creating the view, making it the window's content, and ending when its texts are + set. Rendering is not measured: it is the same for every row and is the UI framework's cost, not the + library's. +- **Culture switch**: `[IterationSetup]` creates the N elements in a window; the benchmark switches the + culture (`Localizer.Current.CurrentCulture`, or `TranslationSource.Instance.Culture` for its baseline), + processes the dispatcher's pending work and checks the last element's text. +- **Avalonia** runs on the headless platform: `[GlobalSetup]` starts a `HeadlessUnitTestSession` once, every + benchmark body runs through its dispatcher, and the window is a headless window. Headless replaces only + the operating system window, input, compositor and message loop; compiled XAML, controls, the property + system, bindings and the library are the same code as in a desktop app, and binding updates after + `PropertyChanged` run on the UI thread without the dispatcher. The numbers therefore show the library's + cost, but not a full frame of a real app. +- **WPF** has no headless mode and uses a real `Window`, shown with `ShowActivated = false` outside the + visible screen area. The benchmarks are marked `[STAThread]` so BenchmarkDotNet runs them on an STA + thread, and pending dispatcher work is processed with a `DispatcherFrame`. + +Alternatives: a scaling run per form (mostly parallel lines, three times the slowest part of the suite, +and the arguments form would need a library-free baseline for live switching with arguments); building the +elements in code and calling `ProvideValue` directly (not what a consumer app +runs, and WPF would need a hand-written service provider); parsing XAML at run time +(`AvaloniaRuntimeXamlLoader`, `XamlReader.Parse`), which no consumer app does; a real Avalonia desktop +window (its timing includes the operating system and rendering, which vary between runs and are not the +library's cost). + +### Regression detection with github-action-benchmark, two series + +The action reads one BenchmarkDotNet JSON file (`tool: benchmarkdotnet`) and compares only the mean time. +Each project writes its own report (`--join` merges the classes within a project), so one script merges the +three and derives the allocation series; the job then feeds the action twice: + +``` + Core / Avalonia / WPF *-report-full-compressed.json (one per project) + | + v + scripts/convert-benchmark-results.ps1 + +--> merged.json (the three Benchmarks arrays, the first report's host info) + | --> github-action-benchmark tool: benchmarkdotnet "Time" + | alert-threshold 150%, fail-on-alert false, summary-always true + | + +--> allocations.json (name + AllocatedBytes per benchmark, customSmallerIsBetter format) + --> github-action-benchmark tool: customSmallerIsBetter "Allocations" + alert-threshold 125%, summary-always true + fail-on-alert: true, false while the PR has label accept-allocation-increase +``` + +Benchmark names include their class's namespace, which differs per project, so names from the three reports +cannot collide. + +- **Pull request**: `auto-push: false`; the action compares against the latest entry on the data branch + and writes the result to the job summary. `comment-on-alert` is on only when the pull request comes from + this repository, because a fork's token cannot comment. +- **Push to `main`**: `auto-push: true` appends the entry to the `benchmark-data` branch. +- An allocation baseline of 0 bytes: the action computes the ratio against 0; the conversion script writes + allocated bytes plus 1 so a 0 -> N change becomes an alert instead of a division by zero (spec: "a + benchmark that allocated nothing on `main` SHALL fail on any allocation"). + +Alternatives: a hand-written comparer (new tool to maintain); failing on slower times (the +speed of shared CI machines varies, so the check would fail at random); BenchmarkDotNet's own +`--statisticalTest` against a baseline run (needs both versions run on the same machine, which is left out). + +### Accepting a deliberate allocation increase with a label + +The "Allocations" step sets +`fail-on-alert: ${{ !contains(github.event.pull_request.labels.*.name, 'accept-allocation-increase') }}`, so +a labeled pull request still gets the alert in the job summary (and the comment) but passes. The workflow's +`pull_request` trigger lists `types: [opened, synchronize, reopened, labeled, unlabeled]`, so adding or +removing the label re-runs the check. Nothing else is needed: the merged push to `main` records the higher +values, which become the baseline. + +- Only people with triage access or more can label a pull request, so a contributor cannot accept their own + increase; a maintainer does it after reading the alert. +- `labeled` and `unlabeled` re-run the whole CI workflow, not only the benchmark job; the existing + `concurrency` group cancels the run in progress. Accepted: labeling is rare. + +Alternatives: a separate workflow on `labeled` only (a second place that runs the benchmarks); a commit +message marker (contributors could set it themselves); a hand-edited baseline file (has to be maintained). + +### Three parallel benchmark jobs and one report job, on windows-latest + +``` + benchmark-changes --> benchmarks (matrix: Core | Avalonia | WPF) --> benchmark-report + run? yes/no each: restore, build, run, upload JSON merge, two action steps, + latest/ on main +``` + +- **`benchmark-changes`**: a small job that compares the changed files with the pull request's base (or, + on a push, with the previous commit of `main`) using `git diff --name-only`, and outputs whether + anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, + `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`. The other two jobs run only when it + did (spec: "Change without code is not benchmarked"). A skipped job counts as passed for a required check, + so documentation pull requests are not blocked. +- **`benchmarks`**: a three-entry matrix, one suite per entry, all on `windows-latest`. Each builds and runs + its own project and uploads its BenchmarkDotNet artifacts. The wall-clock time is the slowest suite's, + expected to be WPF. One runner type keeps one comparable history. `timeout-minutes: 15`, so a hanging + benchmark cannot hold up CI for long. +- **`benchmark-report`**: needs the matrix, downloads the three JSON reports, runs + `scripts/convert-benchmark-results.ps1` and the two `github-action-benchmark` steps, and on `main` writes + `latest/` to the data branch. Only this job has `contents: write` (push to the data branch) and + `pull-requests: write` (comments); the workflow default stays `contents: read`. + +All of this runs in parallel to the build jobs, so it lengthens CI only if it takes longer than the slowest +of them. + +Alternatives: one job running the three suites one after the other (the wall-clock time is their sum); a +third-party action to detect changed paths (`git diff` does the same without a new dependency). + +### Publishing through the data branch + +The `benchmark-data` branch holds the action's history (`history/data.js` and its chart `index.html`) and, +written by the job on `main` in the same push, the latest BenchmarkDotNet HTML and markdown reports under +`latest/`. `deploy-docs` builds the Pages content from two sources: + +``` + package-and-docs --(artifact: docs-site)--+ + +--> deploy-docs: site + benchmarks/ --> Pages + benchmark-data branch (latest/, history/) -+ +``` + +- `package-and-docs` uploads the DocFX site as an ordinary artifact instead of the Pages artifact. +- `deploy-docs` (main only) downloads it, checks out `benchmark-data` into `benchmarks/` inside the site, + then runs `upload-pages-artifact` and `deploy-pages`. It needs `package-and-docs` and `benchmark-report` + with `if: always() && needs.package-and-docs.result == 'success'`, so a failed or skipped benchmark run + leaves the last successful results on the site (spec: "Benchmark run fails on main"). +- A short page, `docs/guides/benchmarks.md`: a table of each benchmark and its baseline, one sentence on + the Ratio and Allocated columns, and links to + `https://rent-a-developer.github.io/ResXLocalization/benchmarks/latest/` and `.../benchmarks/history/` by + absolute URL, because those files are not DocFX content and relative links would fail + `--warningsAsErrors`. It joins `docs/toc.yml` under Guides. No worked examples: the results answer whether + the library is fast enough by themselves. + +Alternative: commit result markdown into `docs/` from CI (a bot commit on `main` per push). + +## Risks / Trade-offs + +- [The times measured on shared CI machines vary by 10-30% between runs] -> a slower time only raises an + alert, at 150%; only more allocations fail the check. +- [Allocation numbers of UI benchmarks include small allocations of the UI framework, such as dispatcher + work] -> the 25% tolerance, which also leaves room for a feature that adds a small allocation; if a UI + benchmark still fails and passes at random, move it to a third series that alerts without failing. +- [WPF under BenchmarkDotNet's in-process toolchain: STA thread and dispatcher] -> the first task is a + feasibility check that runs one WPF benchmark end to end before the rest is written. +- [Avalonia headless session under the in-process toolchain] -> the same feasibility check for Avalonia. +- [The two frameworks may expect different change notifications for an indexer binding (`"Item[]"` or + `"Item"`), and Avalonia's compiled bindings must accept an indexer on a `Source={x:Static ...}` source] -> + the feasibility check also switches the culture once on a `TranslationSource` binding in each framework, + and the self-check catches a baseline that does not update. +- [The first run on `main` has no history] -> the action does not alert without a previous entry; the first + pull request after merge compares against it. +- [The benchmarks could still make CI slower than the 10-minute target] -> the estimate is about 2 minutes + of restore and build plus 1-4 minutes of measuring per suite. The feasibility check measures the real time + of one suite; if a suite takes over 10 minutes, the suite is cut further before the rest is written. The + first full CI run confirms it. +- [The 10,000-element scaling benchmarks are slow, mostly in WPF] -> those classes run one invocation per + iteration (`InvocationCount = 1`, needed anyway for `[IterationSetup]`). +- [Short iterations make the times vary more between runs] -> accepted: times only alert, at 150%, and + allocations are not affected. +- [A pull request that changes a benchmark's name loses its history] -> accepted; the renamed benchmark is + treated as new and does not fail. +- [The data branch grows with each push to `main`] -> the action keeps at most `max-items-in-chart` entries + (set to 200). + +## Migration Plan + +1. Merge the projects, the job and the docs page. +2. The first push to `main` creates the `benchmark-data` branch and the first entry. +3. Pull requests compare against it from then on. + +Rollback: remove the job and restore the Pages artifact upload in `package-and-docs`; the branch can stay. diff --git a/openspec/changes/add-benchmarks/proposal.md b/openspec/changes/add-benchmarks/proposal.md new file mode 100644 index 0000000..0c2fba4 --- /dev/null +++ b/openspec/changes/add-benchmarks/proposal.md @@ -0,0 +1,71 @@ +# Proposal + +## Why + +ResXLocalization has no performance measurements. Nothing catches a change that makes lookups slower or +allocate more, and consumers cannot see what the library costs them: whether it makes their app slow, and how +it behaves with thousands of localized UI elements and a live culture switch. A small benchmark suite in CI +answers both questions without turning into a performance study. + +## What Changes + +- A BenchmarkDotNet suite under a new `benchmarks/` folder that measures the library's main features against + the code a consumer would write without it - the ResX designer class and hand-written code over it: + - Code lookups through `Localizer.Current.Get` (typed key, scoped key, search-all across several registered + resource managers, enum), with and without `MessageArguments`, against the designer class's property and + `string.Format` over it. + - ICU MessageFormat through `ResXLocalization.MessageFormat` (one message with named arguments), against calling the + MessageFormat library directly. + - XAML lookups for Avalonia (headless) and WPF: the cost per element of each extension form + (`LocalizeExtension`, with `Argument` children, `LocalizeEnumExtension`), measured on 100 elements, + against `{x:Static}` of the designer class. Every UI benchmark is a XAML view compiled by the build and + shown in a window, as in a consumer app. + - Scaling with plain `LocalizeExtension`: opening a view with N localized elements, and switching the + culture with N live bindings, for N = 100 and 10,000, against `{x:Static}` and a hand-written + localization source with an indexer. +- CI jobs that run the suite, in under 10 minutes and in parallel with the existing build jobs, on every pull + request and every push to `main` that changes code, fail a pull request when + a benchmark allocates more than 125% of its bytes on `main`, and reports a time regression above a + threshold without failing. A maintainer can accept a deliberate allocation increase for one pull request + with the label `accept-allocation-increase`, which turns that failure into an alert. +- A published benchmark report on the existing GitHub Pages documentation site: the latest results with + runner details, and the history of every benchmark over the commits on `main`. +- A documentation page that explains what the numbers mean for an application. +- No other localization library is benchmarked; a comparison with one can be added later. +- Benchmarks run on the JIT on .NET 10 only; Native AOT is not benchmarked. + +No shipped package, public API or runtime behavior changes. + +## Capabilities + +### New Capabilities + +- `performance-benchmarks`: the repository's benchmark suite, what it measures and against which baselines, + how CI detects performance regressions with it, and how its results are published. + +### Modified Capabilities + +None. No spec exists yet, and no shipped behavior changes. + +## Impact + +- New projects, one per suite: `benchmarks/ResXLocalization.Core.Benchmarks` (Core and MessageFormat), + `benchmarks/ResXLocalization.Avalonia.Benchmarks` and + `benchmarks/ResXLocalization.WPF.Benchmarks` (Windows only), plus a `benchmarks/Directory.Build.props` and + the code the three share. +- `ResXLocalization.slnx` and `ResXLocalization.NonWindows.slnf` gain the benchmark projects. +- `Directory.Packages.props` gains `BenchmarkDotNet`, referenced only by the benchmark projects, so no + package's dependencies change. +- `.github/workflows/ci.yml` gains a benchmark job, and the documentation deployment includes its results. + Results are stored in two places: + - **CI artifacts** of every run, pull requests and `main`: the raw BenchmarkDotNet reports (JSON, HTML, + markdown, logs), for inspecting that one run. + - **The `benchmark-data` branch**, written only by pushes to `main`, one commit per push: `history/data.js` + with the time and allocation results of the last 200 commits on `main` (the baseline pull requests + compare against, and the data of the history chart), the chart page `history/index.html`, and the HTML + and markdown reports of the latest run under `latest/`. The branch holds no code and is never merged; + the documentation deployment copies it into the site. Writing it needs `contents: write` for the + benchmark job. +- `docs/` gains a benchmarks page and a table-of-contents entry. +- The third-party GitHub Action `benchmark-action/github-action-benchmark` becomes a CI dependency, pinned by + commit SHA like the existing actions. diff --git a/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md new file mode 100644 index 0000000..12ceb85 --- /dev/null +++ b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md @@ -0,0 +1,241 @@ +# Spec Delta + +## Purpose + +Measures what ResXLocalization costs an application and detects performance regressions in the library, so +maintainers see a slower change before it ships and consumers can judge the overhead before they adopt it. + +## ADDED Requirements + +### Requirement: Baselines are the code an application writes without the library +Every benchmark SHALL be reported as a ratio to a baseline that does the same work without ResXLocalization: +the ResX designer class's property, hand-written code over it, or a direct call of the MessageFormat library. +No other localization library SHALL be benchmarked. + +#### Scenario: Every result has a baseline +- **WHEN** the suite runs +- **THEN** every result table contains one baseline row that uses no ResXLocalization API, and every other + row shows its ratio to that row + +#### Scenario: Baseline and library produce the same text +- **WHEN** a benchmark is checked before it is measured +- **THEN** its baseline and its ResXLocalization rows produce the same text for the same key, culture and + arguments + +### Requirement: Code lookups are benchmarked against the ResX designer class +The suite SHALL measure `Localizer.Current.Get` for a typed key, a scoped key, a search-all key and an enum +value, each with mean time and allocated bytes, against the designer class's property for the same key, read +in the same culture. + +#### Scenario: Typed key lookup reports its overhead +- **WHEN** the code lookup benchmarks run +- **THEN** the results contain a designer-property baseline row and a typed-key `Get` row with the same key + and culture, and the `Get` row shows its ratio to the baseline and its allocated bytes + +### Requirement: Search-all lookups are benchmarked by the number of registered resource managers +The suite SHALL measure a search-all lookup whose key exists only in the last registered resource manager, +for 1, 5 and 20 registered resource managers. + +#### Scenario: Lookup cost grows with the registered managers +- **WHEN** the search-all benchmark runs +- **THEN** the results contain one row per manager count (1, 5 and 20), each finding the key in the last + registered manager + +#### Scenario: Managers without the key are searched in the lookup culture +- **WHEN** the search-all benchmark looks up a key in a specific culture, such as `de-DE` +- **THEN** each registered manager that does not contain the key is searched in that culture, including its + culture fallback chain, before the next manager is tried, as in an application + +### Requirement: Format arguments in code are benchmarked +The suite SHALL measure `Get` with positional arguments (`MessageArguments.FromValues`), named arguments +(`MessageArguments.FromTuples`) and object arguments (`MessageArguments.FromObject`), against +`string.Format` over the designer class's property with the same culture and arguments. + +#### Scenario: Each argument form reports its overhead +- **WHEN** the argument benchmarks run +- **THEN** the results contain one row per argument form and a `string.Format` baseline row with the same + pattern, culture and argument values + +### Requirement: ICU MessageFormat is benchmarked against the MessageFormat library +The suite SHALL measure one lookup formatted by the ICU MessageFormat formatter, for a message with two +named arguments, against formatting the designer class's property directly with a formatter of the +MessageFormat library that has the library's default settings, with the same culture and argument values. +The overhead does not depend on which ICU features a message uses, so no other message is measured. + +#### Scenario: One message with named arguments is measured +- **WHEN** the MessageFormat benchmarks run +- **THEN** the results contain a direct MessageFormat library baseline row and one ResXLocalization row for + the same message with two named arguments, and both produce the same text + +#### Scenario: The overhead of the adapter is visible +- **WHEN** the MessageFormat benchmarks run +- **THEN** each ResXLocalization row shows its ratio to the direct MessageFormat library call for the same + message + +### Requirement: UI benchmarks use XAML compiled by the build, as a consumer app does +Every UI benchmark, its baselines included, SHALL write the markup extensions, bindings and `{x:Static}` in +XAML views that the project's build compiles with the UI framework's own tooling, SHALL create those views +as an application does, and SHALL measure until the view is the content of a window and its texts are set. +Rendering SHALL NOT be part of the measurement. + +#### Scenario: A benchmark row is a compiled view +- **WHEN** a UI benchmark runs +- **THEN** the measured operation creates a view defined in a `.axaml` or `.xaml` file of the benchmark + project, makes it the content of a window, and ends when the view's texts are set + +#### Scenario: XAML is not parsed at runtime +- **WHEN** a UI benchmark creates a view +- **THEN** the view's markup was compiled by the build, and no XAML is parsed from a string or file at run + time + +### Requirement: Avalonia benchmarks run on the headless platform +The Avalonia suite SHALL run on Avalonia's headless platform, which keeps Avalonia's controls, XAML, +property system and bindings and replaces only the operating system window, input and rendering. + +#### Scenario: Avalonia suite runs without a desktop +- **WHEN** the Avalonia suite runs on a CI machine +- **THEN** it opens no operating system window and needs no display + +### Requirement: Each extension form is benchmarked for Avalonia and WPF +For each UI framework, the suite SHALL measure opening 100 text elements that all use one form - +`LocalizeExtension` with a typed key, `LocalizeExtension` with two `Argument` children, or +`LocalizeEnumExtension` - from 10-element views used 10 times in one window, against the same layout with +`{x:Static}` of the designer class's property. + +#### Scenario: Each form reports its cost per element +- **WHEN** the forms benchmark runs for a UI framework +- **THEN** the results contain a `{x:Static}` baseline row and one row per form, each for 100 elements, and + every element shows its expected localized text + +### Requirement: Scaling with many localized elements is benchmarked +For each UI framework, the suite SHALL measure opening a view with N localized text elements, and switching +the culture while N localized text elements are live, for N = 100 and 10,000, reporting time and +allocated bytes. The N elements SHALL use `LocalizeExtension` with a typed key and SHALL come from a +compiled view with 10 text elements, used N / 10 times in one window. + +#### Scenario: Culture switch updates every element +- **WHEN** the culture-switch benchmark runs with N live elements +- **THEN** the measurement ends only after all N elements show the text of the new culture + +#### Scenario: Results are reported per element count +- **WHEN** the scaling benchmarks run +- **THEN** the results contain one row per N for opening a view and one row per N for a culture switch + +### Requirement: Scaling benchmarks have library-free baselines +Opening a view SHALL be measured against the same views with `{x:Static}`, and a culture switch against the +same views with every element bound in XAML to a hand-written localization source: a singleton with a string indexer over the designer class's +resource manager that raises a change notification for its indexer when its culture changes. + +#### Scenario: Opening a view is compared with static texts +- **WHEN** the open-view benchmark runs with N elements +- **THEN** the results contain a `{x:Static}` baseline row and a `LocalizeExtension` row for the same N + +#### Scenario: Culture switch is compared with a hand-written source +- **WHEN** the culture-switch benchmark runs with N live elements +- **THEN** the results contain a hand-written-source baseline row and a `LocalizeExtension` row for the same + N, and both end with all N elements showing the text of the new culture + +### Requirement: Benchmarks verify their own results +Every benchmark SHALL check, once before it is measured, that its operation produces the expected text, and +SHALL stop the run with an error when it does not. + +#### Scenario: A broken lookup does not produce a number +- **WHEN** a benchmarked lookup returns the lookup failure text instead of the expected text +- **THEN** the benchmark run fails with an error that names the benchmark, and no result is reported for it + +### Requirement: CI runs the suite on pull requests and on main +CI SHALL run the full suite on every pull request to `main` and on every push to `main` that changes code, +build configuration or the benchmarks, on one fixed runner type and .NET 10 on the JIT. From the start of +the benchmark jobs to the published comparison, the run SHALL take under 10 minutes of wall-clock time. + +#### Scenario: Pull request runs the suite +- **WHEN** a pull request to `main` that changes code is opened or updated +- **THEN** CI runs every benchmark, attaches the raw results to the run, and the benchmark comparison is + reported within 10 minutes of the benchmark jobs starting + +#### Scenario: Change without code is not benchmarked +- **WHEN** a pull request or a push to `main` changes only files outside the library sources, the benchmarks, + the build configuration and the CI workflow, such as documentation +- **THEN** the benchmarks do not run, and the benchmark check does not block the pull request + +### Requirement: Measurements run on optimized code +Every benchmark SHALL be measured only after warmup iterations that let the JIT compile its code fully +optimized, so allocations that optimized code avoids, such as objects the JIT allocates on the stack, are +not counted. + +#### Scenario: Allocations match an application's +- **WHEN** a benchmark's allocations are measured +- **THEN** they are measured after its warmup iterations, on the same optimized code a long-running + application runs + +### Requirement: CI fails a pull request whose benchmarks allocate more +CI SHALL fail a pull request when any benchmark allocates more than 125% of the bytes per operation of the +same benchmark in the latest result from `main`. A benchmark that allocated nothing on `main` SHALL fail on +any allocation. A benchmark with no result on `main` SHALL NOT fail the check. + +#### Scenario: Extra allocation fails the check +- **WHEN** a pull request makes a benchmark allocate more than 125% of its bytes per operation on `main` +- **THEN** the benchmark check fails and names the benchmark with both allocation values + +#### Scenario: Small growth does not fail the check +- **WHEN** a benchmark's bytes per operation grow by at most 25% over `main`, such as a new feature adding a small allocation +- **THEN** the check does not fail because of that benchmark + +#### Scenario: New benchmark does not fail the check +- **WHEN** a pull request adds a benchmark that has no result on `main` +- **THEN** the check does not fail because of that benchmark + +### Requirement: A label accepts an allocation increase for one pull request +While a pull request carries the label `accept-allocation-increase`, CI SHALL report allocation increases +above the threshold as alerts, as it reports time regressions, instead of failing. Adding or removing the +label SHALL re-run the benchmark check. The label SHALL NOT change the check of any other pull request. + +#### Scenario: Labeled pull request passes with an alert +- **WHEN** a pull request with the label `accept-allocation-increase` makes a benchmark allocate more than + 125% of its bytes per operation on `main` +- **THEN** the benchmark check passes, and the job summary lists the benchmark with both allocation values + +#### Scenario: Label is removed +- **WHEN** the label is removed from a pull request whose benchmarks allocate more than the threshold +- **THEN** the benchmark check runs again and fails + +#### Scenario: Accepted increase becomes the new baseline +- **WHEN** a labeled pull request is merged and its push to `main` records new results +- **THEN** later pull requests compare against the increased allocation values + +### Requirement: CI reports time regressions without failing +CI SHALL report every benchmark whose mean time exceeds 150% of the latest result from `main`, in the run's +job summary and, where the run may comment, on the pull request. A time regression alone SHALL NOT fail the +check. + +#### Scenario: Slower benchmark is reported +- **WHEN** a pull request makes a benchmark take more than 150% of its time on `main` +- **THEN** the job summary lists the benchmark with both times, and the check still passes if no allocation + increased + +#### Scenario: Pull request from a fork +- **WHEN** the pull request comes from a fork and the run cannot comment +- **THEN** the comparison appears in the job summary and the run does not fail because it cannot comment + +### Requirement: Benchmark results are published on the documentation site +Each push to `main` SHALL publish, on the GitHub Pages documentation site, that commit's results as tables +with the runner's hardware and runtime details, and a history chart per benchmark over the commits on +`main`. A failed benchmark run SHALL NOT block the documentation deployment. + +#### Scenario: Consumer reads the latest results +- **WHEN** a push to `main` finishes and the documentation site is deployed +- **THEN** the site has that commit's result tables, with the hardware and runtime they were measured on, and + the history chart, both reachable from the documentation's table of contents + +#### Scenario: Benchmark run fails on main +- **WHEN** the benchmark job fails on a push to `main` +- **THEN** the documentation site is still deployed, with the benchmark results of the last successful run + +### Requirement: The documentation links the results +The documentation SHALL have one short page that links the latest results and the history chart and says, +for each benchmark, what it is compared with, and that the Ratio column is the time relative to that +baseline. + +#### Scenario: Reader finds the results +- **WHEN** a developer opens the benchmarks page from the documentation's table of contents +- **THEN** it links the latest results and the history chart and names each benchmark's baseline diff --git a/openspec/changes/add-benchmarks/tasks.md b/openspec/changes/add-benchmarks/tasks.md new file mode 100644 index 0000000..93cf095 --- /dev/null +++ b/openspec/changes/add-benchmarks/tasks.md @@ -0,0 +1,62 @@ +# Tasks + +## 1. Project setup and feasibility check + +- [ ] 1.1 Add `BenchmarkDotNet` to `Directory.Packages.props` (current stable version) and verify `dotnet restore ResXLocalization.slnx` succeeds +- [ ] 1.2 Create `benchmarks/Directory.Build.props` (imports the root props, adds `NoWarn` `CS1574;CS1591` with the same comment as the samples, and compiles `benchmarks/Shared/*.cs` into every benchmark project) and the three console projects `benchmarks/ResXLocalization.Core.Benchmarks` (`$(CurrentTargetFramework)`, Core and MessageFormat), `benchmarks/ResXLocalization.Avalonia.Benchmarks` (`$(CurrentTargetFramework)`, Avalonia) and `benchmarks/ResXLocalization.WPF.Benchmarks` (`$(CurrentWindowsTargetFramework)`, `UseWPF`), each with the library, Core and generator `ProjectReference`s the samples use and a `BenchmarkSwitcher` `Program`; verify all three build in Release +- [ ] 1.3 Add the three projects to `ResXLocalization.slnx` under `/04_benchmarks/`, and Core and Avalonia (not WPF) to `ResXLocalization.NonWindows.slnf`; verify `dotnet build ResXLocalization.NonWindows.slnf -c Release` and `dotnet test ResXLocalization.slnx -c Release` still pass and `dotnet test` runs no benchmark +- [ ] 1.4 Feasibility check: one Avalonia headless benchmark and one WPF `[STAThread]` benchmark that create a compiled XAML view with a `{l:Localize}` `TextBlock`, make it a window's content (headless window in Avalonia, a real `Window` outside the visible screen area in WPF) and check its text, and switch the culture once on a view whose `TextBlock` binds `[key]` on a minimal `TranslationSource` with `Source={x:Static ...}` (in Avalonia with compiled bindings), in their two projects; verify both run end to end with `dotnet run -c Release -- --filter *Feasibility* --inProcess --warmupCount 3 --iterationCount 3 --iterationTime 100` from each project and report time and allocations, record each project's total run time and confirm it leaves a suite of the planned size well under 10 minutes (otherwise cut the suite in the design before going on), then delete the feasibility check classes + +## 2. Resource data and shared configuration + +- [ ] 2.1 Write by hand, in each of the three projects, `Resources/BenchmarkStrings.resx`, `BenchmarkStrings.de.resx` (every key) and a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` emits, with the same project-file wiring as the samples: in Core about 10 keys (plain text, positional and named arguments, one ICU message with two named arguments, enum values) plus one `Filler.resx` with a single other key; in Avalonia and WPF 10 keys including an argument key and the enum keys; verify the generator emits `BenchmarkStringsKeys`, the designer properties compile and the satellites build +- [ ] 2.2 Add the shared `[Config]` to `benchmarks/Shared/` (MemoryDiagnoser; JSON full, GitHub markdown and HTML exporters; one job with `InProcessEmitToolchain`, 3 warmup and 3 measured iterations of 100 ms; `ArtifactsPath` set to `artifacts/benchmarks//` from the repository root) and `[STAThread]` on the WPF project's `Main`, plus the culture setup that sets `CultureInfo.CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture; verify a run of a class in each project writes all three report files under `artifacts/benchmarks//results` and no `BenchmarkDotNet.Artifacts` folder, runs in-process (no generated project is built) and reports 3 warmup and 3 measured iterations +- [ ] 2.3 Add the self-check helper to `benchmarks/Shared/` that `[GlobalSetup]` calls to run each benchmark method once and throw on an unexpected text; verify by temporarily breaking one key that the class fails with an error naming the benchmark and produces no result row + +## 3. Core and MessageFormat benchmarks + +- [ ] 3.1 `CodeLookupBenchmarks`: designer property baseline (`BenchmarkStrings.`), typed key, scoped key, search-all key and enum `Get`, in `de-DE`; verify with `--filter *CodeLookup*` that every row has a Ratio and Allocated value +- [ ] 3.2 `SearchAllBenchmarks` with `[Params(1, 5, 20)]` registered managers: 0, 4 or 19 separate `ResourceManager` instances over `Filler.resx`, then `BenchmarkStrings`, with the key only in `BenchmarkStrings`; verify three rows with rising time +- [ ] 3.3 `ArgumentBenchmarks`: `string.Format(culture, BenchmarkStrings., ...)` baseline, `FromValues`, `FromTuples`, `FromObject`; verify four rows and that the self-check passes for each +- [ ] 3.4 `MessageFormatBenchmarks`: one ICU message with two named arguments (a plural with `{Count}` and `{Name}`); a baseline that formats the designer property with one `new MessageFormatter()` and the same values and culture, against `Get` with `MessageArguments.FromTuples` and `UseMessageFormat()` installed; verify two rows and that the self-check confirms both produce the same text + +## 4. Avalonia benchmarks + +- [ ] 4.1 Headless session helper in the Avalonia project, started once in each class's `[GlobalSetup]` and running each body on its dispatcher; verify two classes in one run share the session without errors +- [ ] 4.2 Four 10-element `.axaml` views, each using one form for all elements and the 10 keys once (`{x:Static res:BenchmarkStrings.}`, `{l:Localize}` typed key, `{l:Localize}` with two `` children bound to the view's `DataContext`, `{l:LocalizeEnum}`), and `AvaloniaFormsBenchmarks`, one row per view (the `{x:Static}` view is the baseline), each opening 10 instances (100 elements) in a `StackPanel` as the headless window's content; verify four rows and that every element's self-check sees its expected text +- [ ] 4.3 Avalonia `TranslationSource`: singleton with a string indexer over `BenchmarkStrings.ResourceManager` and a `Culture` setter that raises the indexer's change notification; verify a compiled-XAML `TextBlock` bound to `[key]` on it updates after a culture change +- [ ] 4.4 Write the 10-element `TranslationSource` `.axaml` view (the 10 keys once) and add `AvaloniaScalingBenchmarks`, reusing the `{l:Localize}` and `{x:Static}` views of 4.2, with `[Params(100, 10000)]`: open a view (N / 10 views in a `StackPanel` as the window's content; baseline the `{x:Static}` views) and culture switch (the views built in `[IterationSetup]`, `InvocationCount = 1`, done when the last element shows the new culture's text; baseline the `TranslationSource` views); verify each operation has a baseline and a library row per N, and the run time of the class + +## 5. WPF benchmarks + +- [ ] 5.1 WPF helpers: a `Window` shown with `ShowActivated = false` outside the visible screen area that hosts the measured view, and a dispatcher flush with `DispatcherFrame`; verify a compiled-XAML view with a `{l:Localize}` `TextBlock` shows its text after one flush +- [ ] 5.2 WPF `TranslationSource`, mirroring 4.3; verify the same way +- [ ] 5.3 The four 10-element `.xaml` views and `WpfFormsBenchmarks`, mirroring 4.2, all `[STAThread]`; verify the same rows as Avalonia and that the self-checks pass +- [ ] 5.4 The `TranslationSource` `.xaml` view and `WpfScalingBenchmarks`, mirroring 4.4; verify the same rows as Avalonia, and that the whole WPF project runs locally in under 6 minutes +- [ ] 5.5 Run the `ui_parity_reviewer` over the Avalonia and WPF benchmark classes and resolve any gap it reports + +## 6. CI integration + +- [ ] 6.1 Add `scripts/convert-benchmark-results.ps1` that reads the three projects' BenchmarkDotNet full JSON reports, writes `merged.json` (their `Benchmarks` arrays, the first report's host info) and `allocations.json` (`customSmallerIsBetter`, allocated bytes + 1 per benchmark); verify on local results that every benchmark of all three projects appears once in each file, with a number +- [ ] 6.2 Add the `benchmark-changes` job to `.github/workflows/ci.yml`: `git diff --name-only` against the pull request's base or the push's previous commit, with the output `run` true when anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`; verify on a documentation-only test pull request that `run` is false and the benchmark jobs are skipped, and on a code change that it is true +- [ ] 6.3 Add the `benchmarks` matrix job (Core, Avalonia, WPF) on `windows-latest`, running only when `benchmark-changes` says so: setup, restore, build and run its project with `--join`, upload its `artifacts/benchmarks//` folder, `timeout-minutes: 15`; verify with `pwsh -File scripts/tidy-code.ps1 -Scope all -Check` and `scripts/verify-line-endings.ps1` +- [ ] 6.4 Add the `benchmark-report` job that needs the matrix: download the three reports, merge with `scripts/convert-benchmark-results.ps1`, run `benchmark-action/github-action-benchmark` pinned by SHA twice (`benchmarkdotnet` "Time" at 150% without failing, `customSmallerIsBetter` "Allocations" at 125% failing, `summary-always`, `comment-on-alert` only for same-repository pull requests, `auto-push` only on `main`, branch `benchmark-data`, `max-items-in-chart: 200`), with job-level `contents: write` and `pull-requests: write`; verify with the lint job +- [ ] 6.5 Make the "Allocations" step's `fail-on-alert` false while the pull request has the label `accept-allocation-increase`, and add `labeled` and `unlabeled` to the `pull_request` trigger's `types` (with `opened`, `synchronize`, `reopened`); verify with the lint job, and on a test pull request that adding and removing the label re-runs the check and turns a failing allocation alert into a passing one and back +- [ ] 6.6 Create the `accept-allocation-increase` label in the repository (maintainer step, with a description saying it accepts a reviewed allocation increase for one pull request); verify it appears in the repository's labels +- [ ] 6.7 On `main`, have `benchmark-report` copy the HTML and markdown reports into `latest/` on the `benchmark-data` branch; verify on a pull-request run that the comparison is reported within 10 minutes of the benchmark jobs starting and its summary shows both series + +## 7. Publishing and documentation + +- [ ] 7.1 Change `package-and-docs` to upload the DocFX site as an ordinary artifact, and `deploy-docs` to need `package-and-docs` and `benchmark-report` with `if: always() && needs.package-and-docs.result == 'success'`, download the site, check out `benchmark-data` into `benchmarks/` inside it, then upload and deploy the Pages artifact; verify the workflow passes the lint job +- [ ] 7.2 Write a short `docs/guides/benchmarks.md` (a table of each benchmark and its baseline, one sentence on the Ratio and Allocated columns, absolute links to `benchmarks/latest/` and `benchmarks/history/`) and add it to `docs/toc.yml` under Guides; verify `dotnet tool run docfx build build/docfx/docfx.json --warningsAsErrors` passes +- [ ] 7.3 Add a "Running the benchmarks" section to `CONTRIBUTING.md` (local command, `--filter`, how CI gates allocations and alerts on time, and when a maintainer applies `accept-allocation-increase`) and the `benchmarks/` row to the project map in `AGENTS.md`; verify the documented command runs as written +- [ ] 7.4 Add an `[Unreleased]` entry in `CHANGELOG.md` for the published benchmark results page; verify the changelog has no dated section added + +## 8. Integration check + +- [ ] 8.1 Run `pwsh -File scripts/pre-commit-gate.ps1` and verify it passes with the benchmark projects included +- [ ] 8.2 After merge, verify the first push to `main` creates the `benchmark-data` branch and the deployed site serves `benchmarks/latest/` and `benchmarks/history/`, and that the next pull request's summary compares against that entry + +## Workflow follow-up + +- Archive the change after the integration check on `main` has passed. From ec08403ba311e8cd21d3458a0a1b4014c7288409 Mon Sep 17 00:00:00 2001 From: David Liebeherr Date: Mon, 26 Oct 2026 02:11:39 +0100 Subject: [PATCH 2/2] test: add the BenchmarkDotNet suite and the CI benchmark job Core, Avalonia and WPF benchmarks against the designer class and {x:Static}, a CI job that fails a pull request on allocation regressions, the published history on GitHub Pages, and the benchmarks docs page. Refs: #53 --- .github/workflows/ci.yml | 297 +++++++++++++++++- AGENTS.md | 1 + CHANGELOG.md | 8 + CONTRIBUTING.md | 30 ++ Directory.Packages.props | 4 + ResXLocalization.NonWindows.slnf | 4 +- ResXLocalization.slnx | 5 + benchmarks/Directory.Build.props | 82 +++++ .../AvaloniaFormsBenchmarks.cs | 71 +++++ .../AvaloniaScalingBenchmarks.cs | 124 ++++++++ .../AvaloniaSession.cs | 108 +++++++ .../BenchmarkApp.cs | 21 ++ .../Program.cs | 8 + ...esXLocalization.Avalonia.Benchmarks.csproj | 27 ++ .../Views/ArgumentsView.axaml | 89 ++++++ .../Views/ArgumentsView.axaml.cs | 12 + .../Views/EnumView.axaml | 18 ++ .../Views/EnumView.axaml.cs | 12 + .../Views/LocalizeView.axaml | 18 ++ .../Views/LocalizeView.axaml.cs | 12 + .../Views/StaticView.axaml | 18 ++ .../Views/StaticView.axaml.cs | 12 + .../Views/TranslationSourceView.axaml | 18 ++ .../Views/TranslationSourceView.axaml.cs | 12 + .../ArgumentBenchmarks.cs | 54 ++++ .../CodeLookupBenchmarks.cs | 45 +++ .../MessageFormatBenchmarks.cs | 57 ++++ .../Program.cs | 8 + .../ResXLocalization.Core.Benchmarks.csproj | 23 ++ .../Resources/Filler.resx | 19 ++ .../SearchAllBenchmarks.cs | 70 +++++ .../Program.cs | 15 + .../ResXLocalization.WPF.Benchmarks.csproj | 27 ++ .../Views/ArgumentsView.xaml | 88 ++++++ .../Views/ArgumentsView.xaml.cs | 9 + .../Views/EnumView.xaml | 18 ++ .../Views/EnumView.xaml.cs | 9 + .../Views/LocalizeView.xaml | 18 ++ .../Views/LocalizeView.xaml.cs | 9 + .../Views/StaticView.xaml | 18 ++ .../Views/StaticView.xaml.cs | 9 + .../Views/TranslationSourceView.xaml | 18 ++ .../Views/TranslationSourceView.xaml.cs | 9 + .../WpfFormsBenchmarks.cs | 74 +++++ .../WpfScalingBenchmarks.cs | 118 +++++++ .../WpfWindow.cs | 115 +++++++ benchmarks/Shared/ArgumentValues.cs | 14 + benchmarks/Shared/BenchmarkConfig.cs | 77 +++++ benchmarks/Shared/BenchmarkCulture.cs | 30 ++ benchmarks/Shared/BenchmarkProgram.cs | 36 +++ benchmarks/Shared/ExpectedViewTexts.cs | 87 +++++ benchmarks/Shared/Priority.cs | 20 ++ .../Resources/BenchmarkStrings.Designer.cs | 234 ++++++++++++++ .../Shared/Resources/BenchmarkStrings.de.resx | 73 +++++ .../Shared/Resources/BenchmarkStrings.resx | 73 +++++ benchmarks/Shared/SelfCheck.cs | 76 +++++ benchmarks/Shared/TranslationSource.cs | 47 +++ docs/guides/benchmarks.md | 26 ++ docs/toc.yml | 2 + openspec/changes/add-benchmarks/design.md | 175 ++++++++--- .../specs/performance-benchmarks/spec.md | 8 +- openspec/changes/add-benchmarks/tasks.md | 83 +++-- scripts/convert-benchmark-results.ps1 | 88 ++++++ 63 files changed, 2907 insertions(+), 83 deletions(-) create mode 100644 benchmarks/Directory.Build.props create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml create mode 100644 benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/Program.cs create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx create mode 100644 benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs create mode 100644 benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs create mode 100644 benchmarks/Shared/ArgumentValues.cs create mode 100644 benchmarks/Shared/BenchmarkConfig.cs create mode 100644 benchmarks/Shared/BenchmarkCulture.cs create mode 100644 benchmarks/Shared/BenchmarkProgram.cs create mode 100644 benchmarks/Shared/ExpectedViewTexts.cs create mode 100644 benchmarks/Shared/Priority.cs create mode 100644 benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs create mode 100644 benchmarks/Shared/Resources/BenchmarkStrings.de.resx create mode 100644 benchmarks/Shared/Resources/BenchmarkStrings.resx create mode 100644 benchmarks/Shared/SelfCheck.cs create mode 100644 benchmarks/Shared/TranslationSource.cs create mode 100644 docs/guides/benchmarks.md create mode 100644 scripts/convert-benchmark-results.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b8898d..ad8274a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,15 @@ on: branches: - main - release/** + # labeled and unlabeled re-run the checks when a maintainer adds or removes accept-allocation-increase, + # which turns a failing allocation alert of the benchmark comparison into a passing one and back. They + # re-run the whole workflow; the concurrency group below cancels the run in progress, and labeling is rare. + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled push: branches: - main @@ -25,6 +34,8 @@ env: AOT_OUTPUT: artifacts/package-aot AVALONIA_CONSUMER_PROJECT: tests/package-consumption/AvaloniaConsumer/AvaloniaConsumer.csproj COMBINED_CONSUMER_PROJECT: tests/package-consumption/CombinedConsumer/CombinedConsumer.csproj + BENCHMARK_DATA_BRANCH: benchmark-data + BENCHMARK_REPORTS: artifacts/benchmarks CONFIGURATION: Release CONSUMER_DIRECTORY: tests/package-consumption DOCFX_CONFIG: build/docfx/docfx.json @@ -332,12 +343,16 @@ jobs: echo "version=${version}" >> "${GITHUB_OUTPUT}" echo "Packed version ${version}" + # An ordinary artifact rather than the Pages artifact: deploy-docs adds the benchmark results to the + # site before it builds the Pages artifact. Only deploy-docs (pushes to main) consumes it. - name: Upload documentation site - # The Pages artifact is only consumed by deploy-docs (pushes to main); skip the upload on PRs. if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + name: docs-site path: ${{ env.DOCS_SITE }} + if-no-files-found: error + retention-days: 1 - name: Upload NuGet packages uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -350,6 +365,239 @@ jobs: retention-days: 30 compression-level: 0 + # Decides whether the benchmarks run: only when something they measure, or how they are run, changed, so a + # documentation-only pull request is not held up by them. A skipped job counts as passed for a required + # check. Without a previous commit to compare with - a new branch, a tag, a manual run - they run. + benchmark-changes: + name: Detect benchmark changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run: ${{ steps.changes.outputs.run }} + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Compare the changed files + id: changes + shell: bash + env: + BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + run: | + if [ -z "${BASE}" ] || ! git cat-file -e "${BASE}^{commit}" 2> /dev/null; then + echo "No previous commit to compare with; the benchmarks run." + echo "run=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + pattern='^(src/|benchmarks/|build/|Directory\.[^/]+$|global\.json$|\.github/workflows/ci\.yml$|scripts/convert-benchmark-results\.ps1$)' + + if git diff --name-only "${BASE}" HEAD | grep -qE "${pattern}"; then + echo "run=true" >> "${GITHUB_OUTPUT}" + else + echo "Nothing the benchmarks measure changed; they are skipped." + echo "run=false" >> "${GITHUB_OUTPUT}" + fi + + # One suite per entry, in parallel with the build jobs, all on one runner type so that the history stays + # comparable. Every benchmark class writes its own report, so each table's Ratio column is against that + # class's baseline; --join would merge a suite's classes into one table with one baseline. The suites check + # their own results before measuring and exit non-zero when a benchmark fails or none ran, so a broken + # benchmark fails here instead of missing from the report. + benchmarks: + name: Benchmarks (${{ matrix.suite }}) + runs-on: windows-latest + timeout-minutes: 15 + needs: benchmark-changes + if: needs.benchmark-changes.outputs.run == 'true' + strategy: + fail-fast: false + matrix: + suite: + - Core + - Avalonia + - WPF + env: + BENCHMARK_PROJECT: benchmarks/ResXLocalization.${{ matrix.suite }}.Benchmarks/ResXLocalization.${{ matrix.suite }}.Benchmarks.csproj + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: 10.0.x + dotnet-quality: ga + + - name: Cache NuGet packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ env.NUGET_PACKAGES }} + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets', '**/*.slnx', '.config/dotnet-tools.json') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Restore + run: dotnet restore ${{ env.BENCHMARK_PROJECT }} --configfile ${{ env.NUGET_CONFIG }} + + - name: Build + run: dotnet build ${{ env.BENCHMARK_PROJECT }} --configuration ${{ env.CONFIGURATION }} --no-restore /p:ContinuousIntegrationBuild=true + + - name: Run benchmarks + run: dotnet run --project ${{ env.BENCHMARK_PROJECT }} --configuration ${{ env.CONFIGURATION }} --no-build -- --filter "*" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: benchmark-results-${{ matrix.suite }} + path: ${{ env.BENCHMARK_REPORTS }}/ResXLocalization.${{ matrix.suite }}.Benchmarks/ + if-no-files-found: error + retention-days: 30 + + # Compares the results with the latest results from main and reports both series in the job summary: a time + # above 150% is an alert, an allocation above 125% fails a pull request - unless a maintainer accepted the + # increase with the label accept-allocation-increase. On main the results are appended to the history on the + # benchmark-data branch, with this run's reports under latest/, which deploy-docs publishes. Only this job + # can write. + benchmark-report: + name: Benchmark report + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: benchmarks + permissions: + contents: write + pull-requests: write + env: + BENCHMARK_SERIES: artifacts/benchmarks/series + # A fork's token cannot comment, so only a pull request from this repository comments on an alert; the + # job summary has the comparison either way. + COMMENT_ON_ALERT: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + IS_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Download benchmark results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: benchmark-results-* + path: ${{ env.BENCHMARK_REPORTS }} + + - name: Merge benchmark results + run: pwsh -NoProfile -NonInteractive -File scripts/convert-benchmark-results.ps1 -ReportDirectory ${{ env.BENCHMARK_REPORTS }} -OutputDirectory ${{ env.BENCHMARK_SERIES }} + + # The action needs the data branch to exist. The first push to main creates it, with one commit and no + # results; a pull request before that has nothing to compare with. + - name: Find the benchmark data branch + id: data-branch + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + if git ls-remote --exit-code --heads origin "${BENCHMARK_DATA_BRANCH}" > /dev/null; then + echo "exists=true" >> "${GITHUB_OUTPUT}" + elif [ "${IS_MAIN}" = "true" ]; then + tree="$(gh api "repos/${GITHUB_REPOSITORY}/git/trees" -f 'tree[][path]=README.md' -f 'tree[][mode]=100644' -f 'tree[][type]=blob' -f 'tree[][content]=Benchmark results of main, written by CI. This branch holds no code and is never merged.' --jq .sha)" + commit="$(gh api "repos/${GITHUB_REPOSITORY}/git/commits" -f message="Create the benchmark data branch" -f tree="${tree}" --jq .sha)" + gh api "repos/${GITHUB_REPOSITORY}/git/refs" -f ref="refs/heads/${BENCHMARK_DATA_BRANCH}" -f sha="${commit}" > /dev/null + echo "exists=true" >> "${GITHUB_OUTPUT}" + else + echo "::notice title=No benchmark baseline::main has no benchmark results yet, so there is nothing to compare with." + echo "exists=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Compare times + if: steps.data-branch.outputs.exists == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: Time + tool: benchmarkdotnet + output-file-path: ${{ env.BENCHMARK_SERIES }}/times.json + gh-pages-branch: ${{ env.BENCHMARK_DATA_BRANCH }} + benchmark-data-dir-path: history + github-token: ${{ github.token }} + auto-push: false + alert-threshold: "150%" + fail-on-alert: false + comment-on-alert: ${{ env.COMMENT_ON_ALERT }} + summary-always: true + max-items-in-chart: 200 + + # skip-fetch-gh-pages: the step above fetched the branch and committed to it locally, and fetching it + # again would be refused as a non-fast-forward update. Only a pull request fails on an alert: on main the + # increase was reviewed, and the new values become the baseline. + - name: Compare allocations + if: steps.data-branch.outputs.exists == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: Allocations + tool: customSmallerIsBetter + output-file-path: ${{ env.BENCHMARK_SERIES }}/allocations.json + gh-pages-branch: ${{ env.BENCHMARK_DATA_BRANCH }} + benchmark-data-dir-path: history + github-token: ${{ github.token }} + auto-push: false + skip-fetch-gh-pages: true + alert-threshold: "125%" + fail-on-alert: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'accept-allocation-increase') }} + comment-on-alert: ${{ env.COMMENT_ON_ALERT }} + summary-always: true + max-items-in-chart: 200 + + # Each of the two steps above committed to the local data branch. Squash them, add this run's HTML and + # markdown reports under latest/ (one per benchmark class) with an index page, and push one commit. + - name: Publish to the benchmark data branch + if: env.IS_MAIN == 'true' && steps.data-branch.outputs.exists == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + reports="$(mktemp -d)" + links="" + for suite in Core Avalonia WPF; do + for html in "${BENCHMARK_REPORTS}/benchmark-results-${suite}/results"/*-report.html; do + report="$(basename "${html}" -report.html)" + cp "${html}" "${reports}/${report}.html" + cp "${html%.html}-github.md" "${reports}/${report}.md" + links="${links}
  • ${suite}: ${report##*.} (markdown)
  • " + done + done + + { + echo '' + echo '' + echo 'ResXLocalization benchmark results' + echo '' + echo '

    ResXLocalization benchmark results

    ' + echo "

    Commit ${GITHUB_SHA}. Each report starts with the hardware and runtime it was measured on.

    " + echo '
      ' + echo "${links}" + echo '
    ' + echo '

    History of every benchmark on main

    ' + echo '' + echo '' + } > "${reports}/index.html" + + git switch "${BENCHMARK_DATA_BRANCH}" + git reset --soft HEAD~2 + rm -rf latest + mkdir latest + cp "${reports}"/* latest/ + git add history latest + git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" commit --quiet -m "Benchmark results for ${GITHUB_SHA}" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "${BENCHMARK_DATA_BRANCH}" + # Prove the source generator's Roslyn floor forever: consume the packed packages with ONLY the .NET 8 # SDK. A generator compiled against a newer Roslyn than the .NET 8 compiler would emit CS9057, skip # generation, and fail the consumer build on the missing …Keys types. @@ -549,14 +797,19 @@ jobs: retention-days: 14 # Publish the DocFX site to GitHub Pages on every push to main (requires Pages to be set to - # "GitHub Actions" as the source in the repository settings). + # "GitHub Actions" as the source in the repository settings), with the benchmark-data branch under + # benchmarks/. It waits for benchmark-report without depending on it: when the benchmarks fail or are + # skipped, the site is deployed with the last results the branch has. deploy-docs: name: Deploy docs to GitHub Pages runs-on: ubuntu-latest timeout-minutes: 10 - needs: package-and-docs - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - package-and-docs + - benchmark-report + if: always() && needs.package-and-docs.result == 'success' && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: + contents: read pages: write id-token: write environment: @@ -564,6 +817,40 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Download documentation site + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: docs-site + path: ${{ env.DOCS_SITE }} + + - name: Find the benchmark data branch + id: data-branch + shell: bash + run: | + if git ls-remote --exit-code --heads "https://github.com/${GITHUB_REPOSITORY}.git" "${BENCHMARK_DATA_BRANCH}" > /dev/null; then + echo "exists=true" >> "${GITHUB_OUTPUT}" + else + echo "exists=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Add the benchmark results + if: steps.data-branch.outputs.exists == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ env.BENCHMARK_DATA_BRANCH }} + path: ${{ env.DOCS_SITE }}/benchmarks + persist-credentials: false + + - name: Remove the checkout metadata + if: steps.data-branch.outputs.exists == 'true' + shell: bash + run: rm -rf "${DOCS_SITE}/benchmarks/.git" + + - name: Upload the Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ${{ env.DOCS_SITE }} + - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5 diff --git a/AGENTS.md b/AGENTS.md index 2a41825..907a0f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,7 @@ supports neither). | `src/ResXLocalization.WPF` | The same for WPF. Windows-only, no Native AOT | | `samples/` | Two runnable showcases, one per UI framework. Excluded from coverage | | `tests/` | Five suites: Core and `tests/ResXLocalization.MessageFormat.Tests` (both on both frameworks), the generator, and one sample-test suite per UI framework | +| `benchmarks/` | Three BenchmarkDotNet suites - Core with MessageFormat, Avalonia headless, WPF - each against the code an application writes without the library, and `Shared/`, compiled into all three. Not test projects; CI runs them and gates on allocations | | `tests/package-consumption/` | Package-only consumers. The repository's `Directory.Build.*` deliberately does **not** apply there | | `build/` | The shared resx and packaging targets, and the DocFX configuration | | `scripts/` | Every check and workflow, as PowerShell 7 | diff --git a/CHANGELOG.md b/CHANGELOG.md index 28cd0bf..63044cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- A BenchmarkDotNet suite under `benchmarks/` that measures code lookups, ICU MessageFormat and the Avalonia + and WPF markup extensions against the `.resx` designer class. +- A [benchmark results page](https://rent-a-developer.github.io/ResXLocalization/guides/benchmarks.html) in + the documentation: what each lookup, markup extension and culture switch costs compared with the `.resx` + designer class, with the latest results and their history on `main`. + ## [2.0.0] - 2026-10-20 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f00a0d..6d501e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,6 +179,36 @@ starts and answers wrongly. `scripts/verify-package-aot.ps1` publishes the packa The WPF package is exempt — WPF does not support Native AOT. +## Running the benchmarks + +`benchmarks/` holds three BenchmarkDotNet suites: `ResXLocalization.Core.Benchmarks` (Core and +MessageFormat), `ResXLocalization.Avalonia.Benchmarks` (headless) and `ResXLocalization.WPF.Benchmarks` +(Windows only). Each compares the library with the code an application writes without it, see +[`docs/guides/benchmarks.md`](docs/guides/benchmarks.md). Run one suite from the repository root, all of its +benchmarks or the classes `--filter` selects: + +```shell +dotnet run --project benchmarks/ResXLocalization.Core.Benchmarks -c Release +dotnet run --project benchmarks/ResXLocalization.Core.Benchmarks -c Release -- --filter "*CodeLookup*" +``` + +The reports land in `artifacts/benchmarks//results`. Every class checks once, before it measures, +that each benchmark produces the expected text, and stops with an error naming the benchmark when it does +not. + +CI runs the three suites on every pull request and every push to `main` that changes code, and compares +them with the latest results from `main`: + +- **More allocations fail the check**: a benchmark that allocates more than 125% of its bytes on `main`, or + anything at all where it allocated nothing. +- **A slower time is an alert, not a failure**: above 150% of the time on `main` it is listed in the job + summary and commented on the pull request. Shared CI machines vary too much in speed to fail on time. + +When an allocation increase is deliberate - a feature that has to allocate - a maintainer reviews it and adds +the label `accept-allocation-increase` to the pull request. The check then reports the increase and passes, +and the merged values become the baseline for every later pull request. Removing the label makes the check +fail again. + ## Public API The four runtime projects track their public surface with diff --git a/Directory.Packages.props b/Directory.Packages.props index 21248f4..b454bc1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -49,6 +49,10 @@ + + + + + + + + + Exe + + + + + $(NoWarn);CS1574;CS1591 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs new file mode 100644 index 0000000..932a4e8 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs @@ -0,0 +1,71 @@ +using Avalonia.Controls; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// Opening 100 text elements that all use one form - 10 instances of a 10-element view as a headless window's +/// content - against the same layout with {x:Static} of the designer class's properties. +/// +/// +/// 50 warmup iterations instead of the configuration's 3: an iteration here is one operation, and the tiered JIT +/// promotes a method only after 30 calls, so with fewer warmups the measured rows ran partly unoptimized code +/// and the first rows looked slower than the later ones (see also the call-counting delay in +/// benchmarks/Directory.Build.props). +/// +[Config(typeof(BenchmarkConfig))] +[WarmupCount(50)] +public class AvaloniaFormsBenchmarks +{ + private const int ViewCount = 10; + + private Window window = null!; + + [GlobalCleanup] + public void Cleanup() => AvaloniaSession.Run(this.window.Close); + + [IterationCleanup] + public void Idle() => AvaloniaSession.EmptyWindow(this.window); + + [Benchmark] + public StackPanel Localize() => this.Open(static () => new LocalizeView()); + + [Benchmark] + public StackPanel LocalizeEnum() => this.Open(static () => new EnumView()); + + [Benchmark] + public StackPanel LocalizeWithArguments() => this.Open(static () => new ArgumentsView()); + + [GlobalSetup] + public void Setup() + { + AvaloniaSession.Start(); + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = AvaloniaSession.Run(AvaloniaSession.ShowWindow); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Repeat( + name switch + { + nameof(LocalizeWithArguments) => ExpectedViewTexts.GermanWithArguments, + nameof(LocalizeEnum) => ExpectedViewTexts.GermanEnum, + _ => ExpectedViewTexts.German, + }, + ViewCount + ), + AvaloniaSession.GetTextsOfAllTextBlocks + ); + } + + [Benchmark(Baseline = true)] + public StackPanel Static() => this.Open(static () => new StaticView()); + + private StackPanel Open(Func createView) => + AvaloniaSession.Run(() => AvaloniaSession.Open(this.window, ViewCount, createView)); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs new file mode 100644 index 0000000..d69d3d6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs @@ -0,0 +1,124 @@ +using Avalonia.Controls; +using Avalonia.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// Opening a view with {l:Localize} elements, against {x:Static}, and +/// switching the culture while they are live, against the hand-written . The +/// elements are / 10 instances of a 10-element view in one headless window. +/// +/// +/// Every iteration is one operation (InvocationCount = 1), which the culture switch needs for its +/// [IterationSetup] and which keeps the 10,000-element rows short. 5 measured iterations instead of the +/// configuration's 15, because an iteration of a 10,000-element row takes up to a second plus its setup and +/// cleanup. +/// +[Config(typeof(BenchmarkConfig))] +[CategoriesColumn] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[InvocationCount(1)] +[IterationCount(5)] +public class AvaloniaScalingBenchmarks +{ + private const string OpenCategory = "Open"; + + private const string SwitchCategory = "Switch"; + + private Window window = null!; + + [Params(100, 10_000)] + public int ElementCount { get; set; } + + private int ViewCount => this.ElementCount / 10; + + [GlobalCleanup] + public void Cleanup() => AvaloniaSession.Run(this.window.Close); + + [IterationCleanup] + public void Idle() => AvaloniaSession.EmptyWindow(this.window); + + [Benchmark] + [BenchmarkCategory(OpenCategory)] + public StackPanel OpenLocalize() => this.Open(static () => new LocalizeView()); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(OpenCategory)] + public StackPanel OpenStatic() => this.Open(static () => new StaticView()); + + [GlobalSetup] + public void Setup() + { + AvaloniaSession.Start(); + Localizer.Current.ClearResourceManagers(); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = AvaloniaSession.Run(AvaloniaSession.ShowWindow); + + SelfCheck.Verify( + this, + name => + ExpectedViewTexts.Repeat( + name.StartsWith(SwitchCategory, StringComparison.Ordinal) + ? ExpectedViewTexts.English + : ExpectedViewTexts.German, + this.ViewCount + ), + AvaloniaSession.GetTextsOfAllTextBlocks + ); + } + + [IterationSetup(Targets = [nameof(SwitchLocalize)])] + public void SetupLocalizeViews() => this.SetupViews(static () => new LocalizeView()); + + [IterationSetup(Targets = [nameof(SwitchTranslationSource)])] + public void SetupTranslationSourceViews() => this.SetupViews(static () => new TranslationSourceView()); + + [Benchmark] + [BenchmarkCategory(SwitchCategory)] + public StackPanel SwitchLocalize() => + this.Switch(static () => Localizer.Current.CurrentCulture = BenchmarkCulture.English); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(SwitchCategory)] + public StackPanel SwitchTranslationSource() => + this.Switch(static () => TranslationSource.Instance.Culture = BenchmarkCulture.English); + + private StackPanel Open(Func createView) => + AvaloniaSession.Run(() => AvaloniaSession.Open(this.window, this.ViewCount, createView)); + + /// + /// Builds the views of a culture switch in German, outside the measurement. The previous iteration's views + /// are removed first, so that only these views are live when BenchmarkDotNet collects garbage before the + /// iteration. The culture is set on the UI thread, because views an earlier benchmark left may still be + /// subscribed and update their texts; the CurrentUICulture it sets there ends with the call, and none + /// of these views reads it. + /// + /// Creates one view. + private void SetupViews(Func createView) => + AvaloniaSession.Run(() => + { + this.window.Content = null; + BenchmarkCulture.Use(BenchmarkCulture.German); + _ = AvaloniaSession.Open(this.window, this.ViewCount, createView); + }); + + /// + /// Switches the culture and processes the dispatcher's pending binding work, down to + /// , which is above layout and rendering, as in WPF. Avalonia updates + /// a binding on the UI thread as soon as its source raises PropertyChanged, so every element shows the + /// new text when this returns; the self-check confirms it for all of them. + /// + /// Switches the culture. + /// The panel with the views. + private StackPanel Switch(Action switchCulture) => + AvaloniaSession.Run(() => + { + switchCulture(); + Dispatcher.UIThread.RunJobs(DispatcherPriority.Normal); + return (StackPanel)this.window.Content!; + }); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs new file mode 100644 index 0000000..c57e9e6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs @@ -0,0 +1,108 @@ +using System.Diagnostics.CodeAnalysis; +using Avalonia.Controls; +using Avalonia.Headless; +using Avalonia.LogicalTree; +using Avalonia.Threading; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// The one headless Avalonia session of the process, shared by every benchmark class, and the helpers the +/// benchmark bodies use on its dispatcher thread. +/// +internal static class AvaloniaSession +{ + private static HeadlessUnitTestSession? session; + + /// + /// Empties , collects the garbage and runs the dispatcher's pending work, outside + /// the measurement, as an application's dispatcher is idle between two user actions; without this, the views + /// an earlier operation created would still be subscribed to Localizer.Current in the next one. + /// + /// The window to empty. + [SuppressMessage( + "Major Code Smell", + "S1215:\"GC.Collect\" should not be called", + Justification = "Weak subscriptions are only removed once their subscribers were collected." + )] + public static void EmptyWindow(Window window) => + Run(() => + { + window.Content = null; + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Dispatcher.UIThread.RunJobs(); + }); + + /// Reads the text of every below , on the UI thread. + /// The panel a benchmark returned. + /// The texts, in logical-tree order. + public static IReadOnlyList GetTextsOfAllTextBlocks(ILogical root) => + Run(() => + (IReadOnlyList) + [ + .. root.GetLogicalDescendants() + .OfType() + .Select(static textBlock => textBlock.Text ?? string.Empty), + ] + ); + + /// + /// Puts new views into a and makes it the content of + /// . Avalonia applies a binding's value as soon as it is bound, so the texts are set + /// when this returns. + /// + /// The window. + /// The number of views. + /// Creates one view. + /// The panel with the views. + public static StackPanel Open(Window window, int viewCount, Func createView) + { + var panel = new StackPanel(); + + for (var index = 0; index < viewCount; index++) + { + panel.Children.Add(createView()); + } + + window.Content = panel; + return panel; + } + + /// + /// Runs on the session's dispatcher thread and waits for it. The call flows the + /// caller's execution context, and with it , + /// so the culture is set on the calling thread: a culture set inside ends with it. + /// + /// The type of the result. + /// The work to run on the UI thread. + /// The result of . + public static T Run(Func body) => session!.Dispatch(body, CancellationToken.None).GetAwaiter().GetResult(); + + /// Runs on the session's dispatcher thread and waits for it. + /// The work to run on the UI thread. + public static void Run(Action body) => session!.Dispatch(body, CancellationToken.None).GetAwaiter().GetResult(); + + /// + /// Shows a headless window: the window every benchmark of a class makes its views the content of. Its + /// DataContext is the one the argument bindings read. + /// + /// The shown window. + public static Window ShowWindow() + { + var window = new Window + { + DataContext = ArgumentValues.Instance, + Height = 600, + Width = 800, + }; + window.Show(); + Dispatcher.UIThread.RunJobs(DispatcherPriority.Normal); + return window; + } + + /// Starts the session, once per process; each class's [GlobalSetup] calls it. + public static void Start() => session ??= HeadlessUnitTestSession.StartNew(typeof(BenchmarkApp)); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs new file mode 100644 index 0000000..9bab7a5 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs @@ -0,0 +1,21 @@ +using Avalonia; +using Avalonia.Headless; +using Avalonia.Themes.Fluent; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// The headless application every Avalonia benchmark runs in. It has the Fluent theme, as a desktop application +/// does, so that every created control gets its theme styles, as WPF's controls get their default theme styles. +/// Templates are only applied by layout, which no benchmark runs. +/// +public sealed class BenchmarkApp : Application +{ + /// The entry point HeadlessUnitTestSession.StartNew calls to configure the application. + /// The configured application builder. + public static AppBuilder BuildAvaloniaApp() => + AppBuilder.Configure().UseHeadless(new() { UseHeadlessDrawing = true }); + + /// + public override void Initialize() => this.Styles.Add(new FluentTheme()); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs new file mode 100644 index 0000000..9d77a1a --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +internal static class Program +{ + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj b/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj new file mode 100644 index 0000000..73423cd --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj @@ -0,0 +1,27 @@ + + + + + true + $(CurrentTargetFramework) + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml new file mode 100644 index 0000000..be4f542 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs new file mode 100644 index 0000000..0d9db18 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key and two <l:Argument> children bound to the DataContext. +public partial class ArgumentsView : UserControl +{ + public ArgumentsView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml new file mode 100644 index 0000000..e43a911 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs new file mode 100644 index 0000000..f9429e7 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:LocalizeEnum} of a different Priority member. +public partial class EnumView : UserControl +{ + public EnumView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml new file mode 100644 index 0000000..b3c7f8d --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs new file mode 100644 index 0000000..314f7d6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key. +public partial class LocalizeView : UserControl +{ + public LocalizeView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml new file mode 100644 index 0000000..f0723d9 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs new file mode 100644 index 0000000..3ceda12 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {x:Static} of the designer class's property: the baseline of the forms and of opening a view. +public partial class StaticView : UserControl +{ + public StaticView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml new file mode 100644 index 0000000..4d89e91 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs new file mode 100644 index 0000000..a18f005 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text binds the indexer of the hand-written TranslationSource: the baseline of the culture switch. +public partial class TranslationSourceView : UserControl +{ + public TranslationSourceView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs new file mode 100644 index 0000000..900b852 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs @@ -0,0 +1,54 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// A typed-key lookup with the two format arguments of in each +/// form, against +/// over the designer class's property. +/// +[Config(typeof(BenchmarkConfig))] +public class ArgumentBenchmarks +{ + private static readonly ArgumentValues Values = ArgumentValues.Instance; + + [Benchmark] + public string GetWithFromObject() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromObject(new { Values.Name, Values.Count }) + ); + + [Benchmark] + public string GetWithFromTuples() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromTuples((nameof(Values.Name), Values.Name), (nameof(Values.Count), Values.Count)) + ); + + [Benchmark] + public string GetWithFromValues() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromValues(Values.Name, Values.Count) + ); + + [GlobalSetup] + public void Setup() + { + BenchmarkCulture.Use(BenchmarkCulture.German); + + var expected = string.Format( + BenchmarkCulture.German, + ExpectedViewTexts.Text(BenchmarkCulture.German, "UserMessages"), + Values.Name, + Values.Count + ); + SelfCheck.Verify(this, _ => expected); + } + + [Benchmark(Baseline = true)] + public string StringFormatDesignerProperty() => + string.Format(BenchmarkCulture.German, BenchmarkStrings.UserMessages, Values.Name, Values.Count); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs new file mode 100644 index 0000000..f103c9a --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// One per lookup mode, in de-DE, against the +/// designer class's property. +/// +[Config(typeof(BenchmarkConfig))] +public class CodeLookupBenchmarks +{ + [Benchmark] + public string GetEnum() => Localizer.Current.Get(Priority.Priority1); + + [Benchmark] + public string GetStringKeyScoped() => + Localizer.Current.Get("Greeting", resourceManager: BenchmarkStringsKeys.ResourceManager); + + [Benchmark] + public string GetStringKeySearchAll() => Localizer.Current.Get("Greeting"); + + [Benchmark] + public string GetTypedKey() => Localizer.Current.Get(BenchmarkStringsKeys.Greeting); + + [Benchmark(Baseline = true)] + public string ResXDesignerProperty() => BenchmarkStrings.Greeting; + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Text( + BenchmarkCulture.German, + name == nameof(GetEnum) ? "Enum_Priority_Priority1" : "Greeting" + ) + ); + } +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs new file mode 100644 index 0000000..b2ab4b1 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs @@ -0,0 +1,57 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.MessageFormat; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// One ICU message with two named arguments, through the localizer with UseMessageFormat() installed, +/// against formatting the designer class's property with the MessageFormat library directly. Both sides use a +/// formatter with the library's defaults, so pattern caching is the same, and the ratio is the cost of +/// ResXLocalization.MessageFormat itself: building the argument dictionary, the culture substitution and the +/// call through the localizer. +/// +[Config(typeof(BenchmarkConfig))] +public class MessageFormatBenchmarks +{ + private static readonly ArgumentValues Values = ArgumentValues.Instance; + + private readonly Jeffijoe.MessageFormat.MessageFormatter messageFormatter = new(); + + private readonly Dictionary values = new(StringComparer.Ordinal) + { + [nameof(Values.Count)] = Values.Count, + [nameof(Values.Name)] = Values.Name, + }; + + /// Restores the default formatter for the classes that run after this one in the same process. + [GlobalCleanup] + public void Cleanup() => Localizer.Current.MessageFormatter = StringFormatMessageFormatter.Instance; + + [Benchmark] + public string GetWithMessageFormat() => + Localizer.Current.Get( + BenchmarkStringsKeys.FilesDeleted, + MessageArguments.FromTuples((nameof(Values.Count), Values.Count), (nameof(Values.Name), Values.Name)) + ); + + [Benchmark(Baseline = true)] + public string MessageFormatterDirect() => + this.messageFormatter.FormatMessage(BenchmarkStrings.FilesDeleted, this.values, BenchmarkCulture.German); + + [GlobalSetup] + public void Setup() + { + BenchmarkCulture.Use(BenchmarkCulture.German); + Localizer.Current.UseMessageFormat(); + + // The expected text is the satellite's pattern formatted by a separate formatter, so both rows have to + // produce what the MessageFormat library makes of the German text. + var expected = new Jeffijoe.MessageFormat.MessageFormatter().FormatMessage( + ExpectedViewTexts.Text(BenchmarkCulture.German, "FilesDeleted"), + this.values, + BenchmarkCulture.German + ); + SelfCheck.Verify(this, _ => expected); + } +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs b/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs new file mode 100644 index 0000000..c8903e2 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +internal static class Program +{ + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj b/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj new file mode 100644 index 0000000..9c4dd1a --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj @@ -0,0 +1,23 @@ + + + + $(CurrentTargetFramework) + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx b/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx new file mode 100644 index 0000000..41a9ee6 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx @@ -0,0 +1,19 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Filler + + diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs new file mode 100644 index 0000000..f51d83f --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs @@ -0,0 +1,70 @@ +using System.Resources; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// A search-all lookup whose key exists only in the last of registered resource +/// managers, against the loop an application writes without the library when it does not know which file holds +/// a key: on each of the same +/// managers, in the same order, until one has it. Both rows do the same resource-manager work, so the ratio is +/// what the library adds to a search. +/// +/// +/// Every manager before the last is a separate over Filler.resx: the +/// localizer tells managers apart by instance, and each instance has its own resource-set cache, so each miss +/// walks the whole culture fallback chain, de-DE -> de -> neutral, as a different file would. +/// +[Config(typeof(BenchmarkConfig))] +public class SearchAllBenchmarks +{ + private ResourceManager[] resourceManagers = []; + + [Params(1, 5, 20)] + public int ManagerCount { get; set; } + + [Benchmark] + public string GetStringKeySearchAll() => Localizer.Current.Get("Greeting"); + + [Benchmark(Baseline = true)] + public string HandWrittenSearchAll() + { + foreach (var resourceManager in this.resourceManagers) + { + if (resourceManager.GetString("Greeting", BenchmarkCulture.German) is { } text) + { + return text; + } + } + + return "Greeting"; + } + + [GlobalSetup] + public void Setup() + { + this.resourceManagers = + [ + .. Enumerable + .Range(1, this.ManagerCount - 1) + .Select(static _ => new ResourceManager( + FillerKeys.ResourceManager.BaseName, + typeof(FillerKeys).Assembly + )), + BenchmarkStringsKeys.ResourceManager, + ]; + + Localizer.Current.ClearResourceManagers(); + + foreach (var resourceManager in this.resourceManagers) + { + Localizer.Current.RegisterResourceManager(resourceManager); + } + + BenchmarkCulture.Use(BenchmarkCulture.German); + + var expected = ExpectedViewTexts.Text(BenchmarkCulture.German, "Greeting"); + SelfCheck.Verify(this, _ => expected); + } +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs new file mode 100644 index 0000000..673b160 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs @@ -0,0 +1,15 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +internal static class Program +{ + /// + /// [STAThread], because the in-process toolchain runs every benchmark on this thread, and WPF + /// objects need an STA thread. + /// + /// The command-line arguments, passed to BenchmarkDotNet as given. + /// 0 when every selected benchmark produced a result, otherwise 1. + [STAThread] + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj b/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj new file mode 100644 index 0000000..1b27056 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj @@ -0,0 +1,27 @@ + + + + + $(CurrentWindowsTargetFramework) + true + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml new file mode 100644 index 0000000..08f7a58 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs new file mode 100644 index 0000000..e34e2d4 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key and two <l:Argument> children bound to the DataContext. +public partial class ArgumentsView : UserControl +{ + public ArgumentsView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml new file mode 100644 index 0000000..a2e6575 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs new file mode 100644 index 0000000..e2da881 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:LocalizeEnum} of a different Priority member. +public partial class EnumView : UserControl +{ + public EnumView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml new file mode 100644 index 0000000..7a794a2 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs new file mode 100644 index 0000000..72bfc05 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key. +public partial class LocalizeView : UserControl +{ + public LocalizeView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml new file mode 100644 index 0000000..25630f5 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs new file mode 100644 index 0000000..36284b8 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {x:Static} of the designer class's property: the baseline of the forms and of opening a view. +public partial class StaticView : UserControl +{ + public StaticView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml new file mode 100644 index 0000000..f13f99e --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs new file mode 100644 index 0000000..d426cb2 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text binds the indexer of the hand-written TranslationSource: the baseline of the culture switch. +public partial class TranslationSourceView : UserControl +{ + public TranslationSourceView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs new file mode 100644 index 0000000..ee6cadd --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs @@ -0,0 +1,74 @@ +using System.Windows; +using System.Windows.Controls; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// Opening 100 text elements that all use one form - 10 instances of a 10-element view as a window's content - +/// against the same layout with {x:Static} of the designer class's properties. +/// +/// +/// 300 warmup iterations instead of the configuration's 3, where Avalonia needs 50: an iteration here is one +/// operation, and WPF ships as ReadyToRun code, which the tiered JIT recompiles to optimized code method by +/// method on a background thread - thousands of WPF methods. With 50 warmups the first row of the process still +/// measured about 2 ms more than the same row run last (Static: 3.4 ms against 1.4 ms); with 300 every row +/// measures the same wherever it runs. See also the call-counting delay in benchmarks/Directory.Build.props. +/// +[Config(typeof(BenchmarkConfig))] +[WarmupCount(300)] +public class WpfFormsBenchmarks +{ + private const int ViewCount = 10; + + private Window window = null!; + + [GlobalCleanup] + public void Cleanup() => this.window.Close(); + + [IterationCleanup] + public void Idle() => WpfWindow.EmptyWindow(this.window); + + [Benchmark] + [STAThread] + public StackPanel Localize() => WpfWindow.Open(this.window, ViewCount, static () => new LocalizeView()); + + [Benchmark] + [STAThread] + public StackPanel LocalizeEnum() => WpfWindow.Open(this.window, ViewCount, static () => new EnumView()); + + [Benchmark] + [STAThread] + public StackPanel LocalizeWithArguments() => + WpfWindow.Open(this.window, ViewCount, static () => new ArgumentsView()); + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = WpfWindow.Show(); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Repeat( + name switch + { + nameof(LocalizeWithArguments) => ExpectedViewTexts.GermanWithArguments, + nameof(LocalizeEnum) => ExpectedViewTexts.GermanEnum, + _ => ExpectedViewTexts.German, + }, + ViewCount + ), + WpfWindow.GetTextsOfAllTextBlocks + ); + } + + [Benchmark(Baseline = true)] + [STAThread] + public StackPanel Static() => WpfWindow.Open(this.window, ViewCount, static () => new StaticView()); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs new file mode 100644 index 0000000..3f43a2d --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs @@ -0,0 +1,118 @@ +using System.Windows; +using System.Windows.Controls; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// Opening a view with {l:Localize} elements, against {x:Static}, and +/// switching the culture while they are live, against the hand-written . The +/// elements are / 10 instances of a 10-element view in one window. +/// +/// +/// Every iteration is one operation (InvocationCount = 1), which the culture switch needs for its +/// [IterationSetup] and which keeps the 10,000-element rows short. 5 measured iterations instead of the +/// configuration's 15, because an iteration of a 10,000-element row takes up to a second plus its setup and +/// cleanup. +/// +[Config(typeof(BenchmarkConfig))] +[CategoriesColumn] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[InvocationCount(1)] +[IterationCount(5)] +public class WpfScalingBenchmarks +{ + private const string OpenCategory = "Open"; + + private const string SwitchCategory = "Switch"; + + private Window window = null!; + + [Params(100, 10_000)] + public int ElementCount { get; set; } + + private int ViewCount => this.ElementCount / 10; + + [GlobalCleanup] + public void Cleanup() => this.window.Close(); + + [IterationCleanup] + public void Idle() => WpfWindow.EmptyWindow(this.window); + + [Benchmark] + [BenchmarkCategory(OpenCategory)] + [STAThread] + public StackPanel OpenLocalize() => WpfWindow.Open(this.window, this.ViewCount, static () => new LocalizeView()); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(OpenCategory)] + [STAThread] + public StackPanel OpenStatic() => WpfWindow.Open(this.window, this.ViewCount, static () => new StaticView()); + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = WpfWindow.Show(); + + SelfCheck.Verify( + this, + name => + ExpectedViewTexts.Repeat( + name.StartsWith(SwitchCategory, StringComparison.Ordinal) + ? ExpectedViewTexts.English + : ExpectedViewTexts.German, + this.ViewCount + ), + WpfWindow.GetTextsOfAllTextBlocks + ); + } + + [IterationSetup(Targets = [nameof(SwitchLocalize)])] + public void SetupLocalizeViews() => this.SetupViews(static () => new LocalizeView()); + + [IterationSetup(Targets = [nameof(SwitchTranslationSource)])] + public void SetupTranslationSourceViews() => this.SetupViews(static () => new TranslationSourceView()); + + [Benchmark] + [BenchmarkCategory(SwitchCategory)] + [STAThread] + public StackPanel SwitchLocalize() => + this.Switch(static () => Localizer.Current.CurrentCulture = BenchmarkCulture.English); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(SwitchCategory)] + [STAThread] + public StackPanel SwitchTranslationSource() => + this.Switch(static () => TranslationSource.Instance.Culture = BenchmarkCulture.English); + + /// + /// Builds the views of a culture switch in German, outside the measurement. The previous iteration's views + /// are removed first, so that only these views are live when BenchmarkDotNet collects garbage before the + /// iteration. + /// + /// Creates one view. + private void SetupViews(Func createView) + { + this.window.Content = null; + BenchmarkCulture.Use(BenchmarkCulture.German); + _ = WpfWindow.Open(this.window, this.ViewCount, createView); + } + + /// + /// Switches the culture and processes the dispatcher's pending binding work, so every element shows the new + /// text when this returns; the self-check confirms it for all of them. + /// + /// Switches the culture. + /// The panel with the views. + private StackPanel Switch(Action switchCulture) + { + switchCulture(); + WpfWindow.Flush(); + return (StackPanel)this.window.Content; + } +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs new file mode 100644 index 0000000..9895e4f --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs @@ -0,0 +1,115 @@ +using System.Diagnostics.CodeAnalysis; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Threading; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// The window the WPF benchmarks make their views the content of, and the helpers the benchmark bodies use. +/// WPF has no headless mode, so it is a real window, shown without activation outside the visible screen +/// area. Every member runs on the benchmark thread, which is an STA thread because Main and the +/// benchmarks are [STAThread]. +/// +internal static class WpfWindow +{ + /// + /// Empties , collects the garbage and lets the dispatcher go idle, outside the + /// measurement. WPF's weak event managers remove the listeners of collected bindings only when the dispatcher + /// is idle, which an application's dispatcher is between two user actions; without this, every view an + /// earlier operation created would still be a listener of Localizer.Current in the next one. + /// + /// The window to empty. + [SuppressMessage( + "Major Code Smell", + "S1215:\"GC.Collect\" should not be called", + Justification = "The weak event managers can only remove listeners whose bindings were collected." + )] + public static void EmptyWindow(Window window) + { + window.Content = null; + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Flush(DispatcherPriority.ApplicationIdle); + } + + /// + /// Processes the dispatcher's pending binding work. The frame ends at , + /// which is above layout and rendering, so those are not part of a measurement. + /// + public static void Flush() => Flush(DispatcherPriority.DataBind); + + /// Reads the text of every below . + /// The panel a benchmark returned. + /// The texts, in logical-tree order. + public static IReadOnlyList GetTextsOfAllTextBlocks(DependencyObject root) => + [.. GetLogicalDescendants(root).OfType().Select(static textBlock => textBlock.Text)]; + + /// + /// Puts new views into a , makes it the content of + /// and processes the pending binding work, so the texts are set when this returns. + /// + /// The window. + /// The number of views. + /// Creates one view. + /// The panel with the views. + public static StackPanel Open(Window window, int viewCount, Func createView) + { + var panel = new StackPanel(); + + for (var index = 0; index < viewCount; index++) + { + _ = panel.Children.Add(createView()); + } + + window.Content = panel; + Flush(); + return panel; + } + + /// + /// Shows a window outside the visible screen area, without activating it. Its DataContext is the one + /// the argument bindings read. + /// + /// The shown window. + public static Window Show() + { + var window = new Window + { + DataContext = ArgumentValues.Instance, + Height = 600, + Left = -32_000, + ShowActivated = false, + ShowInTaskbar = false, + Top = -32_000, + Width = 800, + WindowStartupLocation = WindowStartupLocation.Manual, + WindowStyle = WindowStyle.None, + }; + window.Show(); + Flush(); + return window; + } + + private static void Flush(DispatcherPriority priority) + { + var frame = new DispatcherFrame(); + _ = Dispatcher.CurrentDispatcher.BeginInvoke(priority, new Action(() => frame.Continue = false)); + Dispatcher.PushFrame(frame); + } + + private static IEnumerable GetLogicalDescendants(DependencyObject root) + { + foreach (var child in LogicalTreeHelper.GetChildren(root).OfType()) + { + yield return child; + + foreach (var descendant in GetLogicalDescendants(child)) + { + yield return descendant; + } + } + } +} diff --git a/benchmarks/Shared/ArgumentValues.cs b/benchmarks/Shared/ArgumentValues.cs new file mode 100644 index 0000000..b8b17ae --- /dev/null +++ b/benchmarks/Shared/ArgumentValues.cs @@ -0,0 +1,14 @@ +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The two format arguments of every lookup with arguments: the values the Core argument benchmarks pass, and +/// the view model the two <l:Argument> children of every element in ArgumentsView bind to. +/// +public sealed class ArgumentValues +{ + /// Gets the one instance the benchmarks use. + public static ArgumentValues Instance { get; } = new(); + + public int Count { get; } = 3; + public string Name { get; } = "Ada"; +} diff --git a/benchmarks/Shared/BenchmarkConfig.cs b/benchmarks/Shared/BenchmarkConfig.cs new file mode 100644 index 0000000..9216e52 --- /dev/null +++ b/benchmarks/Shared/BenchmarkConfig.cs @@ -0,0 +1,77 @@ +using System.Reflection; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The one configuration every benchmark class uses, through [Config(typeof(BenchmarkConfig))], locally +/// and in CI. +/// +/// +/// +/// One short job: 3 warmup and 15 measured iterations of 100 ms, instead of BenchmarkDotNet's 500 ms and its +/// automatic choice of the iteration count. 15 iterations keep the mean time stable enough for the alert at 150% +/// in classes that run one operation per iteration, which is every UI class; the scaling classes, whose +/// 10,000-element operations are slow, lower it. Allocations, which fail the check, do not depend on the number +/// of iterations. The warmups stay, so that every measurement runs on fully optimized code, which allocates some +/// objects on the stack that unoptimized code allocates on the heap. +/// +/// +/// The in-process toolchain runs every benchmark in this process instead of a separately built and started +/// one, which removes a build and a process start per benchmark. +/// +/// +public sealed class BenchmarkConfig : ManualConfig +{ + public BenchmarkConfig() + { + this.AddJob( + Job.Default.WithToolchain(InProcessEmitToolchain.Instance) + .WithWarmupCount(3) + .WithIterationCount(15) + .WithIterationTime(TimeInterval.FromMilliseconds(100)) + ); + this.AddDiagnoser(MemoryDiagnoser.Default); + + // The full JSON report is what CI reads. The GitHub markdown and HTML reports, which CI publishes, come + // from BenchmarkDotNet's default configuration, which every [Config] is merged with; adding them here + // too would make BenchmarkDotNet warn about duplicate exporters. + this.AddExporter(JsonExporter.Full); + + // artifacts/ is where the repository puts everything it generates: .gitignore excludes it and + // scripts/clean-build-artifacts.ps1 deletes it. BenchmarkDotNet adds results/ and the logs below it. + this.ArtifactsPath = Path.Combine( + FindRepositoryRoot(), + "artifacts", + "benchmarks", + Assembly.GetEntryAssembly()!.GetName().Name! + ); + } + + /// Finds the repository root: the nearest directory above the binaries with the solution file. + /// The full path of the repository root. + /// The binaries are not below the repository root. + private static string FindRepositoryRoot() + { + for ( + var directory = new DirectoryInfo(AppContext.BaseDirectory); + directory is not null; + directory = directory.Parent + ) + { + if (File.Exists(Path.Combine(directory.FullName, "ResXLocalization.slnx"))) + { + return directory.FullName; + } + } + + throw new InvalidOperationException( + $"No directory above {AppContext.BaseDirectory} contains ResXLocalization.slnx. Run the benchmarks from their build output in the repository." + ); + } +} diff --git a/benchmarks/Shared/BenchmarkCulture.cs b/benchmarks/Shared/BenchmarkCulture.cs new file mode 100644 index 0000000..edf502c --- /dev/null +++ b/benchmarks/Shared/BenchmarkCulture.cs @@ -0,0 +1,30 @@ +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// The cultures the benchmarks look up in, and the setup that makes every side use the same one. +internal static class BenchmarkCulture +{ + /// The culture the culture-switch benchmarks switch to and from. + public static readonly CultureInfo English = CultureInfo.GetCultureInfo("en"); + + /// + /// The culture every benchmark looks up in, as most applications ship: a specific culture with a satellite + /// for its language only, so every lookup walks de-DE -> de. + /// + public static readonly CultureInfo German = CultureInfo.GetCultureInfo("de-DE"); + + /// + /// Sets the culture of every side a benchmark compares to : the designer class, + /// which reads while its Culture property is unset, + /// and . Setups call it on the thread + /// the benchmark runs on; a culture-switch benchmark instead sets only the side it measures. + /// + /// The culture every side looks up in. + public static void Use(CultureInfo culture) + { + CultureInfo.CurrentUICulture = culture; + Localizer.Current.CurrentCulture = culture; + TranslationSource.Instance.Culture = culture; + } +} diff --git a/benchmarks/Shared/BenchmarkProgram.cs b/benchmarks/Shared/BenchmarkProgram.cs new file mode 100644 index 0000000..23cb137 --- /dev/null +++ b/benchmarks/Shared/BenchmarkProgram.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using BenchmarkDotNet.Running; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// The entry point every benchmark project's Main delegates to. +internal static class BenchmarkProgram +{ + /// + /// Runs the benchmarks of that the command line selects, for example with + /// --filter *CodeLookup*, or all of them without arguments, instead of BenchmarkDotNet's interactive + /// prompt. + /// + /// The benchmark project's assembly. + /// The command-line arguments, passed to BenchmarkDotNet as given. + /// + /// 0 when every selected benchmark produced a result; 1 when one failed, such as a self-check, so that + /// CI fails instead of reporting a table with a missing row, and 1 when nothing ran, such as arguments + /// without --filter, which make BenchmarkDotNet wait for a selection that never comes. + /// + public static int Run(Assembly assembly, string[] args) + { + var summaries = BenchmarkSwitcher + .FromAssembly(assembly) + .Run(args.Length == 0 ? ["--filter", "*"] : args) + .ToList(); + + return + summaries.Count == 0 + || summaries.Exists(static summary => + summary.HasCriticalValidationErrors || summary.Reports.Any(static report => !report.Success) + ) + ? 1 + : 0; + } +} diff --git a/benchmarks/Shared/ExpectedViewTexts.cs b/benchmarks/Shared/ExpectedViewTexts.cs new file mode 100644 index 0000000..1b9b634 --- /dev/null +++ b/benchmarks/Shared/ExpectedViewTexts.cs @@ -0,0 +1,87 @@ +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The texts the self-checks expect, read from the project's BenchmarkStrings resx files: the neutral file +/// for English and the .de satellite for German. A German text is read from the satellite alone, without +/// falling back to the neutral file, so a missing satellite fails the self-check instead of letting every side +/// of a benchmark show English. +/// +internal static class ExpectedViewTexts +{ + /// The keys of every 10-element view except the enum view, element by element. + private static readonly string[] ViewKeys = + [ + "Title", + "Save", + "Cancel", + "Open", + "Close", + "Delete", + "UserMessages", + "Enum_Priority_Priority1", + "Enum_Priority_Priority2", + "Enum_Priority_Priority3", + ]; + + /// + /// Gets the English texts of a view without arguments, after a culture switch. They are the neutral file's, + /// which holds under the invariant culture: there is no + /// .en satellite, so falls back to the neutral file at run + /// time, and reading en without fallback would find nothing. + /// + public static IReadOnlyList English => Texts(CultureInfo.InvariantCulture, ViewKeys); + + /// Gets the German texts of a view without arguments: {x:Static}, {l:Localize}, TranslationSource. + public static IReadOnlyList German => Texts(BenchmarkCulture.German, ViewKeys); + + /// Gets the German texts of the enum view, one per member. + public static IReadOnlyList GermanEnum => + Texts( + BenchmarkCulture.German, + [.. Enum.GetValues().Select(static value => $"Enum_Priority_{value}")] + ); + + /// Gets the German texts of the view whose elements all have the arguments of . + public static IReadOnlyList GermanWithArguments => + [ + .. German.Select(static text => + string.Format( + BenchmarkCulture.German, + text, + ArgumentValues.Instance.Name, + ArgumentValues.Instance.Count + ) + ), + ]; + + /// The texts of instances of a view, one after the other. + /// The texts of one view. + /// The number of views. + /// The texts of all views, in order. + public static IReadOnlyList Repeat(IReadOnlyList view, int viewCount) => + [.. Enumerable.Repeat(view, viewCount).SelectMany(static texts => texts)]; + + /// + /// Reads the text of from the resx file of 's language, or + /// from the neutral file for , without falling back to another file. + /// + /// A culture, such as de-DE, or . + /// The resource key. + /// The text. + /// The file or the key does not exist. + public static string Text(CultureInfo culture, string key) + { + var fileCulture = + culture.IsNeutralCulture || culture.Equals(CultureInfo.InvariantCulture) ? culture : culture.Parent; + var resourceSet = + BenchmarkStrings.ResourceManager.GetResourceSet(fileCulture, createIfNotExists: true, tryParents: false) + ?? throw new InvalidOperationException($"BenchmarkStrings has no resource file for '{fileCulture.Name}'."); + + return resourceSet.GetString(key) + ?? throw new InvalidOperationException($"BenchmarkStrings ({fileCulture.Name}) has no key '{key}'."); + } + + private static string[] Texts(CultureInfo culture, string[] keys) => [.. keys.Select(key => Text(culture, key))]; +} diff --git a/benchmarks/Shared/Priority.cs b/benchmarks/Shared/Priority.cs new file mode 100644 index 0000000..05d8ec0 --- /dev/null +++ b/benchmarks/Shared/Priority.cs @@ -0,0 +1,20 @@ +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The enumeration the enum lookups localize, through the keys Enum_Priority_Priority1 to +/// Enum_Priority_Priority10 of BenchmarkStrings.resx. It has 10 members so that the 10 elements +/// of the enum view each show a different text, as the other views do. +/// +public enum Priority +{ + Priority1 = 1, + Priority2 = 2, + Priority3 = 3, + Priority4 = 4, + Priority5 = 5, + Priority6 = 6, + Priority7 = 7, + Priority8 = 8, + Priority9 = 9, + Priority10 = 10, +} diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs b/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs new file mode 100644 index 0000000..4043ea2 --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs @@ -0,0 +1,234 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace RentADeveloper.ResXLocalization.Benchmarks { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "18.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class BenchmarkStrings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal BenchmarkStrings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings", typeof(BenchmarkStrings).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Cancel. + /// + public static string Cancel { + get { + return ResourceManager.GetString("Cancel", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Close. + /// + public static string Close { + get { + return ResourceManager.GetString("Close", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Delete. + /// + public static string Delete { + get { + return ResourceManager.GetString("Delete", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 1. + /// + public static string Enum_Priority_Priority1 { + get { + return ResourceManager.GetString("Enum_Priority_Priority1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 10. + /// + public static string Enum_Priority_Priority10 { + get { + return ResourceManager.GetString("Enum_Priority_Priority10", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 2. + /// + public static string Enum_Priority_Priority2 { + get { + return ResourceManager.GetString("Enum_Priority_Priority2", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 3. + /// + public static string Enum_Priority_Priority3 { + get { + return ResourceManager.GetString("Enum_Priority_Priority3", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 4. + /// + public static string Enum_Priority_Priority4 { + get { + return ResourceManager.GetString("Enum_Priority_Priority4", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 5. + /// + public static string Enum_Priority_Priority5 { + get { + return ResourceManager.GetString("Enum_Priority_Priority5", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 6. + /// + public static string Enum_Priority_Priority6 { + get { + return ResourceManager.GetString("Enum_Priority_Priority6", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 7. + /// + public static string Enum_Priority_Priority7 { + get { + return ResourceManager.GetString("Enum_Priority_Priority7", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 8. + /// + public static string Enum_Priority_Priority8 { + get { + return ResourceManager.GetString("Enum_Priority_Priority8", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 9. + /// + public static string Enum_Priority_Priority9 { + get { + return ResourceManager.GetString("Enum_Priority_Priority9", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {Count, plural, one {# file deleted} other {# files deleted}} by {Name}. + /// + public static string FilesDeleted { + get { + return ResourceManager.GetString("FilesDeleted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Hello. + /// + public static string Greeting { + get { + return ResourceManager.GetString("Greeting", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Open. + /// + public static string Open { + get { + return ResourceManager.GetString("Open", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Save. + /// + public static string Save { + get { + return ResourceManager.GetString("Save", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Settings. + /// + public static string Title { + get { + return ResourceManager.GetString("Title", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} has {1} new messages. + /// + public static string UserMessages { + get { + return ResourceManager.GetString("UserMessages", resourceCulture); + } + } + } +} diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.de.resx b/benchmarks/Shared/Resources/BenchmarkStrings.de.resx new file mode 100644 index 0000000..8a0e79c --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.de.resx @@ -0,0 +1,73 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Hallo + + + Einstellungen + + + Speichern + + + Abbrechen + + + Öffnen + + + Schließen + + + Löschen + + + {0} hat {1} neue Nachrichten + + + {Count, plural, one {# Datei} other {# Dateien}} von {Name} gelöscht + + + Priorität 1 + + + Priorität 2 + + + Priorität 3 + + + Priorität 4 + + + Priorität 5 + + + Priorität 6 + + + Priorität 7 + + + Priorität 8 + + + Priorität 9 + + + Priorität 10 + + diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.resx b/benchmarks/Shared/Resources/BenchmarkStrings.resx new file mode 100644 index 0000000..1a48b76 --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.resx @@ -0,0 +1,73 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Hello + + + Settings + + + Save + + + Cancel + + + Open + + + Close + + + Delete + + + {0} has {1} new messages + + + {Count, plural, one {# file deleted} other {# files deleted}} by {Name} + + + Priority 1 + + + Priority 2 + + + Priority 3 + + + Priority 4 + + + Priority 5 + + + Priority 6 + + + Priority 7 + + + Priority 8 + + + Priority 9 + + + Priority 10 + + diff --git a/benchmarks/Shared/SelfCheck.cs b/benchmarks/Shared/SelfCheck.cs new file mode 100644 index 0000000..597e196 --- /dev/null +++ b/benchmarks/Shared/SelfCheck.cs @@ -0,0 +1,76 @@ +using System.Reflection; +using BenchmarkDotNet.Attributes; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// Runs every benchmark method of a class once and throws when it does not produce the expected texts. Each +/// class calls it from its [GlobalSetup], so a broken lookup stops that class with an error naming the +/// benchmark, instead of producing a number for the failure text. +/// +internal static class SelfCheck +{ + /// Verifies the benchmark methods of that return the text itself. + /// The benchmark class instance. + /// The text each benchmark method, by name, has to return. + public static void Verify(object benchmarks, Func expected) => + Verify(benchmarks, name => [expected(name)], static result => [result]); + + /// + /// Verifies every benchmark method of , after running the + /// [IterationSetup] methods that apply to it. + /// + /// The type every benchmark method of the class returns. + /// The benchmark class instance. + /// The texts each benchmark method, by name, has to produce, in order. + /// Reads the texts out of what a benchmark method returned. + /// A benchmark method produced other texts. + public static void Verify( + object benchmarks, + Func> expected, + Func> texts + ) + { + var type = benchmarks.GetType(); + var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance); + var iterationSetups = methods + .Select(static method => + (Method: method, SetupAttribute: method.GetCustomAttribute()) + ) + .Where(static setup => setup.SetupAttribute is not null) + .ToList(); + + foreach (var method in methods.Where(static method => method.IsDefined(typeof(BenchmarkAttribute)))) + { + foreach (var setup in iterationSetups) + { + if (setup.SetupAttribute?.Targets is not { Length: > 0 } targets || targets.Contains(method.Name)) + { + _ = setup.Method.Invoke(benchmarks, BindingFlags.DoNotWrapExceptions, null, null, null); + } + } + + var actual = texts((TResult)method.Invoke(benchmarks, BindingFlags.DoNotWrapExceptions, null, null, null)!); + var wanted = expected(method.Name); + + if (actual.Count != wanted.Count) + { + throw new InvalidOperationException( + $"Self-check of {method.DeclaringType!.Name}.{method.Name} failed: expected {wanted.Count} texts, " + + $"but it produced {actual.Count}." + ); + } + + for (var index = 0; index < wanted.Count; index++) + { + if (!string.Equals(actual[index], wanted[index], StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Self-check of {method.DeclaringType!.Name}.{method.Name} failed: text {index + 1} of " + + $"{wanted.Count} is \"{actual[index]}\" instead of \"{wanted[index]}\"." + ); + } + } + } + } +} diff --git a/benchmarks/Shared/TranslationSource.cs b/benchmarks/Shared/TranslationSource.cs new file mode 100644 index 0000000..8d9c393 --- /dev/null +++ b/benchmarks/Shared/TranslationSource.cs @@ -0,0 +1,47 @@ +using System.ComponentModel; +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The usual hand-written way to switch languages live without ResXLocalization, and the baseline of the +/// culture switch: elements bind {Binding [Key], Source={x:Static shared:TranslationSource.Instance}}, +/// and setting makes every such binding read its text again. +/// +public sealed class TranslationSource : INotifyPropertyChanged +{ + private CultureInfo culture = CultureInfo.CurrentUICulture; + + private TranslationSource() { } + + public event PropertyChangedEventHandler? PropertyChanged; + + /// Gets the one instance the views bind to. + public static TranslationSource Instance { get; } = new(); + + /// Gets or sets the culture the indexer looks up in, and notifies every indexer binding. + public CultureInfo Culture + { + get => this.culture; + set + { + this.culture = value; + + // The name that tells every indexer binding to read again: WPF's Binding.IndexerName, "Item[]", in the + // WPF project, the only one with a -windows target framework; Avalonia's "Item" otherwise, because its + // compiled indexer bindings ignore "Item[]". One framework-specific name, not both, and not an empty + // name ("every property changed"), which both frameworks accept but handle at a higher cost: the + // baseline does no more work than an application's own source. +#if WINDOWS + this.PropertyChanged?.Invoke(this, new("Item[]")); +#else + this.PropertyChanged?.Invoke(this, new("Item")); +#endif + } + } + + /// Gets the text of in . + /// The resource key. + /// The text, or when there is none. + public string this[string key] => BenchmarkStrings.ResourceManager.GetString(key, this.culture) ?? key; +} diff --git a/docs/guides/benchmarks.md b/docs/guides/benchmarks.md new file mode 100644 index 0000000..2509d7d --- /dev/null +++ b/docs/guides/benchmarks.md @@ -0,0 +1,26 @@ +# Benchmarks + +Every push to `main` measures what ResXLocalization costs an application, on the same GitHub-hosted Windows +runner, .NET 10 and the just-in-time compiler: + +- [Latest results](https://rent-a-developer.github.io/ResXLocalization/benchmarks/latest/): the tables of the + last commit on `main`, with the hardware and runtime they were measured on. +- [History](https://rent-a-developer.github.io/ResXLocalization/benchmarks/history/): the time and allocations + of every benchmark over the last 200 commits on `main`. + +Every benchmark is compared with the code an application writes without the library: the `.resx` designer +class and hand-written code over it. + +| Benchmark | What it measures | Baseline | +| --- | --- | --- | +| `CodeLookupBenchmarks` | `Localizer.Current.Get` with a typed key, a scoped key, a search-all key and an enum value | The designer class's property, `BenchmarkStrings.Greeting` | +| `SearchAllBenchmarks` | A search-all lookup whose key is in the last of 1, 5 or 20 registered resource files | A hand-written loop that asks the same resource managers, in the same order, until one has the key | +| `ArgumentBenchmarks` | `Get` with `MessageArguments.FromValues`, `FromTuples` and `FromObject` | `string.Format` over the designer class's property | +| `MessageFormatBenchmarks` | `Get` with two named arguments and `UseMessageFormat()` installed | The MessageFormat library's `FormatMessage` over the designer class's property | +| `AvaloniaFormsBenchmarks`, `WpfFormsBenchmarks` | Opening 100 elements that all use `{l:Localize}`, `{l:Localize}` with two `` children, or `{l:LocalizeEnum}` | The same views with `{x:Static}` of the designer class's properties | +| `AvaloniaScalingBenchmarks`, `WpfScalingBenchmarks`, Open | Opening a window with 100 or 10,000 `{l:Localize}` elements | The same views with `{x:Static}` | +| `AvaloniaScalingBenchmarks`, `WpfScalingBenchmarks`, Switch | Switching the culture while 100 or 10,000 `{l:Localize}` elements are live | The same views bound to a hand-written `TranslationSource` with a string indexer | + +The **Ratio** column is a benchmark's time divided by its baseline's, so 1.20 means 20% slower than the code +without the library; **Allocated** is the memory one operation allocates, and **Alloc Ratio** compares it with +the baseline the same way. diff --git a/docs/toc.yml b/docs/toc.yml index 6f082b6..959ab21 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -18,6 +18,8 @@ href: guides/native-aot.md - name: Migrating to 2.0 href: guides/migration-2.0.md + - name: Benchmarks + href: guides/benchmarks.md - name: Reference items: - name: Generated keys diff --git a/openspec/changes/add-benchmarks/design.md b/openspec/changes/add-benchmarks/design.md index 91baad5..4731c74 100644 --- a/openspec/changes/add-benchmarks/design.md +++ b/openspec/changes/add-benchmarks/design.md @@ -56,8 +56,11 @@ Constraints from the repository: ``` benchmarks/ - Directory.Build.props imports root props, NoWarn, compiles Shared/*.cs - Shared/ [Config], self-check helper (compiled into all three) + Directory.Build.props imports root props, NoWarn, compiles Shared/*.cs, links + Shared/Resources/BenchmarkStrings.* into every project + Shared/ [Config], self-check, expected texts, culture setup, Priority, + ArgumentValues, TranslationSource (compiled into all three) + Shared/Resources/ BenchmarkStrings.resx, .de.resx, .Designer.cs (linked into all three) ResXLocalization.Core.Benchmarks Core, MessageFormat $(CurrentTargetFramework) ResXLocalization.Avalonia.Benchmarks Avalonia headless $(CurrentTargetFramework) ResXLocalization.WPF.Benchmarks WPF $(CurrentWindowsTargetFramework) @@ -72,7 +75,9 @@ lookup that doubles in cost moves a UI row by a few percent, below both CI thres formatter installed. - `benchmarks/Directory.Build.props` imports the root props, like `tests/` does, adds the `NoWarn` for `CS1574;CS1591` that samples and tests carry for generated keys, and compiles `benchmarks/Shared/*.cs` into - every benchmark project, so the configuration and the self-check exist once. + every benchmark project, so the configuration, the self-check and the types every suite uses (`Priority`, + `ArgumentValues`, `TranslationSource`) exist once. It also links the one `BenchmarkStrings` resource file + set (see "Resource data") into every project. - All three join `ResXLocalization.slnx` under `/04_benchmarks/`; Core and Avalonia also join the non-Windows filter, WPF does not, as in `src/`. They are built by the existing build jobs, so a broken benchmark fails the build, but they are not test projects and `dotnet test` ignores them. @@ -86,26 +91,36 @@ project for MessageFormat (a project for one class); placing them under `tests/` ### Resource data -- **Core**: `Resources/BenchmarkStrings.resx` with about 10 keys (a plain text, texts with positional and - named arguments, one ICU message with two named arguments, the enum values), a `.de` satellite that translates - them, and no `de-DE` satellite. Every benchmark looks up in `de-DE`, which is how most apps ship: a +- **One resource file set for all three suites**: `benchmarks/Shared/Resources/BenchmarkStrings.resx` with + plain texts, a text with two arguments, one ICU message with two named arguments and the 10 enum keys, a + `.de` file that translates them, and no `de-DE` file. `benchmarks/Directory.Build.props` links them into + every project, which embeds the neutral file and builds its own `de` satellite from the `.de` file; a suite + carries keys only another suite uses, which changes no number (see below). Its `LogicalName` is the + designer class's base name, `RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings`, because a linked + file would otherwise be named after the project's `RootNamespace` and its link path. The designer class + and the generated `BenchmarkStringsKeys` are in the namespace of the code under `Shared/`, which every + benchmark file already imports, so no project needs a using of its own for them. +- **Core**: Every benchmark looks up in `de-DE`, which is how most apps ship: a specific culture with a satellite for its language. The fallback walk `de-DE -> de` is `ResourceManager`'s and costs the baseline and the library the same; it matters for search-all, where every manager without - the key is walked through the whole chain (`de-DE -> de -> neutral`) before the next one is tried, and - that walk is the library's cost. Search-all benchmarks register 0, 4 or 19 + the key is walked through the whole chain (`de-DE -> de -> neutral`) before the next one is tried. That + walk is the cost of any search over several files, so the search-all baseline is a hand-written loop over + the same managers that pays it too (see "Baselines"). Search-all benchmarks register 0, 4 or 19 managers without the key before `BenchmarkStrings`, for 1, 5 or 20 managers in all. They come from one - `Filler.resx` with a single other key: the benchmark creates 19 `ResourceManager` instances over its base - name. The library tells managers apart by instance, and each instance has its own resource-set cache, so + `Filler.resx` with a single other key, in the Core project only, whose `CustomToolNamespace` puts + `FillerKeys` into the benchmark classes' namespace: the benchmark creates 19 `ResourceManager` instances + over its base name. The library tells managers apart by instance, and each instance has its own resource-set cache, so every miss is a real lookup with the full fallback walk, the same work as 19 different files. -- **Avalonia and WPF**: each its own `Resources/BenchmarkStrings.resx` with 10 keys, including an argument - key and the enum keys, with a `.de` satellite. Each 10-element view uses all 10 keys once, so a view has - different texts, as a real one does. +- **Avalonia and WPF**: each 10-element view uses 10 different keys once, + so a view has different texts, as a real one does: seven texts and the first three enum keys, and the + `{l:LocalizeEnum}` view one member each of the shared `Priority` enum, whose 10 members `Priority1` to + `Priority10` exist only for that. The key count does not change the numbers: `ResourceManager` keeps a loaded resource set in a dictionary, so one lookup costs the same with 10 keys or 1,000. Loading the set is a one-time startup cost and is not measured. -Every `BenchmarkStrings.resx` has a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` +`BenchmarkStrings.resx` has a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` emits: a public static property per key that calls `ResourceManager.GetString(name, resourceCulture)`. It is the baseline (see "Baselines"). Typed keys come from the source generator, which handles a resx with a designer, as in the samples. @@ -116,19 +131,42 @@ Visual Studio. The data is fixed, so nothing needs to regenerate it. Alternatives: a script that generates large data sets (1,000 keys, 100-element views), which would run once and then go out of date, and whose size does not change the results; a build-time generator, which would be -new tooling for a fixed data set. +new tooling for a fixed data set; a resource file set per project, as each sample has its own (three copies +of the same texts, two of them identical, and the shared code reached each project's copy only through a +global using of that project's `Resources` namespace). ### BenchmarkDotNet configuration - One `[Config]` shared by all classes: `MemoryDiagnoser`, the JSON (full) and GitHub-markdown and HTML - exporters, and one short job used locally and in CI: - - **3 warmup and 3 measured iterations of 100 ms each**, instead of BenchmarkDotNet's 500 ms and its - automatic choice of more iterations. Fewer, shorter iterations make the mean time vary more between - runs, which is acceptable because time only raises an alert at 150%. Allocations, which fail the check, - do not change with the number of iterations. + exporters, and one short job used locally and in CI. The config adds only the JSON exporter: BenchmarkDotNet + merges every `[Config]` with its default configuration, which already has the other two, and adding them + again makes it warn about duplicate exporters. + - **3 warmup and 15 measured iterations of 100 ms each**, instead of BenchmarkDotNet's 500 ms and its + automatic choice of the iteration count. Every UI class runs one operation per iteration (see "The + dispatcher goes idle between iterations"), so its iterations are its samples: with 3 of them the 99.9% + error was often larger than the mean, and two classes measuring the same 100-element view differed by + 40%. 15 iterations cost seconds per class. The scaling classes have 5 (`[IterationCount(5)]`), because an + iteration of a 10,000-element row takes up to a second plus its setup and cleanup. Time only raises an + alert at 150%; allocations, which fail the check, do not change with the number of iterations. - **The warmups stay**: .NET 10 allocates some objects on the stack, but only in fully optimized code. A benchmark measured before the JIT has optimized it would count allocations a running app does not have (spec: "Measurements run on optimized code"). + - **The forms classes have 50 (Avalonia) and 300 (WPF) warmup iterations**, and the host runs **without + the tiered JIT's call-counting delay** (`System.Runtime.TieredCompilation.CallCountingDelayMs = 0` as a + `RuntimeHostConfigurationOption` in `benchmarks/Directory.Build.props`). One operation there runs + hundreds of UI-framework methods, and a method is promoted to optimized code only after 30 calls and + once no new method was compiled for 100 ms. An iteration is one operation, and every operation compiles + new methods, so the delay kept restarting: with Avalonia's Fluent theme, the first row of the first class + measured three times as slow as the same operation later in the process, with 15 warmups and with 50. + Without the delay and with 50 warmups (more than the 30 calls) every Avalonia row matches the same + operation in the scaling class. WPF ships as ReadyToRun code, which the JIT recompiles to optimized code + on a background thread, thousands of methods: with 50 warmups the first WPF row of the process still + measured about 2 ms more than the same row run last (`Static` 3.4 ms against 1.4 ms; turning the JIT's + tiering and ReadyToRun off brought it to 1.7 ms, a larger gen0 budget or non-concurrent GC changed + nothing). With 300 every WPF row measures the same wherever it runs, for about 10 s more. Dynamic PGO still runs, so code and allocations are a long-running application's; + turning tiered compilation off instead changed the Core suite's allocations (no dynamic PGO, so less + stack allocation). It is the host's configuration because the benchmarks run in-process. The scaling + classes keep 3 warmups: one operation is large enough, and they run after the forms classes. - **In-process toolchain** (`InProcessEmitToolchain`): every benchmark runs in the BenchmarkDotNet host process instead of a separately built and started process. That removes a build and a process start per benchmark; the cost is that benchmarks share one process and its GC state, which does not change these @@ -145,9 +183,23 @@ new tooling for a fixed data set. show whether the cost per element stays the same as N grows. - Each class's `[GlobalSetup]` runs every benchmark method of the class once and throws when the text is not the expected one, which makes BenchmarkDotNet stop the class with an error (spec: "Benchmarks verify their - own results"). + own results"). The expected texts are read from the resx files themselves (`ExpectedViewTexts`): German + from the `.de` satellite alone, without fallback, so a missing satellite fails the check instead of letting + every side show English. - `Program.Main` is `BenchmarkSwitcher.FromAssembly(...).Run(args)`, so local runs can filter with - `--filter`. + `--filter`. Without arguments it runs every benchmark (`--filter *`) instead of BenchmarkDotNet's + interactive prompt, which would wait for input. It returns 1 when a benchmark has no result, so CI fails, + and when nothing ran: arguments without `--filter` reach that prompt, which gets no input. +- **One report per class**, not `--join`: a joined table has one baseline for every class that shares a job, + so in the Core suite every Ratio was against `CodeLookupBenchmarks`' baseline. +- Method names say what a row calls, so a row reads on its own in the history chart: `ResXDesignerProperty`, + `GetTypedKey`, `GetStringKeyScoped`, `GetStringKeySearchAll`, `GetEnum`, `HandWrittenSearchAll`, + `StringFormatDesignerProperty`, + `GetWithFromValues` / `FromTuples` / `FromObject`, `MessageFormatterDirect`, `GetWithMessageFormat`; the UI + classes `Static`, `Localize`, `LocalizeWithArguments`, `LocalizeEnum`, `OpenStatic`, `OpenLocalize`, + `SwitchTranslationSource`, `SwitchLocalize`. +- Every result table has its baseline row, the search-all table included: one hand-written search row per + manager count next to the search-all row (spec: "Every result has a baseline"). ### Baselines @@ -157,14 +209,24 @@ benchmarked (a comparison can be added later as extra rows, outside the allocati | Benchmark | `[Baseline = true]` | | --- | --- | | Code lookup (typed, scoped, search-all, enum) | `BenchmarkStrings.Greeting` (designer property) | -| Code lookup with arguments | `string.Format(culture, BenchmarkStrings.PeopleInvited, count)` | +| Search-all with 1, 5 or 20 managers | `HandWrittenSearchAll`: `GetString(key, culture)` on the same managers, in registration order, until one has the key | +| Code lookup with arguments | `string.Format(culture, BenchmarkStrings.UserMessages, name, count)` | | ICU MessageFormat | `messageFormatter.FormatMessage(BenchmarkStrings.FilesDeleted, values, culture)` | | Extension forms (100 elements) | the same 10-element views with `{x:Static}` | | Opening a view with N elements | the same views with `{x:Static}` | | Culture switch with N elements | the same views with `{Binding [Key], Source={x:Static local:TranslationSource.Instance}}` | +- **Search-all**: the designer property knows which file holds a key, so against it the search-all row + measured the search itself: at 20 managers 65 times the time, nearly all of it `ResourceManager`'s misses, + which any search over 20 files pays. The baseline is the loop an application writes without the library + when it does not know the file: `GetString(key, culture)` on each of the same manager instances, in the same + order the localizer has them, until one returns a value. The ratio is what the library adds to that search. + `CodeLookupBenchmarks` keeps the designer property: with one registered manager its search-all row does the + same work as the property. - **Culture**: the designer class reads `CultureInfo.CurrentUICulture` while its `Culture` property is unset, - so each `[GlobalSetup]` sets `CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture. + so every setup calls `BenchmarkCulture.Use`, which sets `CurrentUICulture`, `Localizer.Current.CurrentCulture` + and `TranslationSource.Instance.Culture` to the same culture. Only the culture-switch benchmarks set one + side directly: the side they measure, so that the other side's notification is not part of the operation. - **ICU**: the baseline uses one `new MessageFormatter()` with the library's defaults, which is what `UseMessageFormat()` creates, so pattern caching is the same on both sides. The ratio is the cost of `ResXLocalization.MessageFormat` itself: building the argument dictionary, the culture substitution and the @@ -173,12 +235,15 @@ benchmarked (a comparison can be added later as extra rows, outside the allocati the MessageFormat library and cost the baseline the same. The adapter's overhead depends only on the number of arguments and whether they are named, so the message has two named arguments, which run both of the adapter's loops (positions and names). -- **`TranslationSource`**: the usual hand-written way to switch languages live, about 20 lines per UI - framework, in each UI project. A singleton implementing `INotifyPropertyChanged` with +- **`TranslationSource`**: the usual hand-written way to switch languages live, about 20 lines, in + `benchmarks/Shared/` for both UI suites. A singleton implementing `INotifyPropertyChanged` with `this[string key] => BenchmarkStrings.ResourceManager.GetString(key, this.Culture)` and a `Culture` setter - that raises `PropertyChanged` for the indexer (`"Item[]"`). Elements bind `[key]` on it with the - framework's normal binding. Its culture-switch row sets `TranslationSource.Instance.Culture`; the library's - row sets `Localizer.Current.CurrentCulture`. + that raises one `PropertyChanged` for the indexer, with the framework's name, chosen by `#if WINDOWS` (only + the WPF project has a `-windows` target framework): `"Item[]"` (`Binding.IndexerName`) in WPF and `"Item"` + in Avalonia, whose compiled indexer bindings ignore `"Item[]"` (found by the feasibility check). Raising + both would make the baseline handle two events; an empty name ("every property changed") works in both + but made the baseline's 10,000-element switch allocate three times as much in WPF. Elements bind `[key]` on it with the framework's normal binding. Its culture-switch row sets + `TranslationSource.Instance.Culture`; the library's row sets `Localizer.Current.CurrentCulture`. Alternatives: raw `ResourceManager.GetString` (the same cost as the designer property, but code consumers do not recognize); Microsoft.Extensions.Localization in the Core suite (rarely used in XAML applications, so it @@ -206,8 +271,8 @@ app: - **WPF** compiles `.xaml` to BAML, and `InitializeComponent` reads it at run time with WPF's XAML reader and its service provider. Reading BAML is a cost consumers pay too; the baseline views pay it as well, so the ratio still shows what `{l:Localize}` adds. -- **Views per UI project**: five 10-element views, written by hand, each using one form for all its - elements: +- **Views per UI project**: five 10-element views under `Views/`, written by hand, each using one form for + all its elements: ``` {x:Static} baseline of both benchmarks @@ -236,7 +301,13 @@ app: culture (`Localizer.Current.CurrentCulture`, or `TranslationSource.Instance.Culture` for its baseline), processes the dispatcher's pending work and checks the last element's text. - **Avalonia** runs on the headless platform: `[GlobalSetup]` starts a `HeadlessUnitTestSession` once, every - benchmark body runs through its dispatcher, and the window is a headless window. Headless replaces only + benchmark body runs through its dispatcher, and the window is a headless window. The application has the + Fluent theme, as a desktop application does, so every control gets its theme styles, as WPF's controls get + their default theme styles. Templates are applied only by layout, which no row runs in either framework; + the library does not depend on them or on the visual tree (it has no attach or detach hooks, and + `DataContext` is inherited through the logical tree), which the self-checks confirm. The session's + dispatch flows the caller's execution context, and with it `CultureInfo.CurrentUICulture`, so the culture + is set on the BenchmarkDotNet thread: a culture set inside a dispatched body is gone in the next one. Headless replaces only the operating system window, input, compositor and message loop; compiled XAML, controls, the property system, bindings and the library are the same code as in a desktop app, and binding updates after `PropertyChanged` run on the UI thread without the dispatcher. The numbers therefore show the library's @@ -244,6 +315,17 @@ app: - **WPF** has no headless mode and uses a real `Window`, shown with `ShowActivated = false` outside the visible screen area. The benchmarks are marked `[STAThread]` so BenchmarkDotNet runs them on an STA thread, and pending dispatcher work is processed with a `DispatcherFrame`. +- **Pending work is processed down to binding priority only**: WPF's frame ends at + `DispatcherPriority.DataBind`, Avalonia's `RunJobs` at `DispatcherPriority.Normal` (Avalonia's `DataBind` is + obsolete). Both are above layout and rendering, which therefore stay out of the measurement in both + frameworks. +- **The dispatcher goes idle between iterations**: every UI class has an `[IterationCleanup]` that empties the + window, collects garbage and runs the dispatcher's idle work, outside the measurement, as an application's + dispatcher is idle between two user actions. WPF removes the weak-event listeners of collected bindings only + at idle priority; without this step every view an earlier operation created was still a listener of + `Localizer.Current`, and a culture switch with 100 elements took about six times as long. Avalonia does the + same for parity. Because of the `[IterationCleanup]`, BenchmarkDotNet runs one operation per iteration in the + forms classes too, like the scaling classes. Alternatives: a scaling run per form (mostly parallel lines, three times the slowest part of the suite, and the arguments form would need a library-free baseline for live switching with arguments); building the @@ -256,15 +338,15 @@ library's cost). ### Regression detection with github-action-benchmark, two series The action reads one BenchmarkDotNet JSON file (`tool: benchmarkdotnet`) and compares only the mean time. -Each project writes its own report (`--join` merges the classes within a project), so one script merges the -three and derives the allocation series; the job then feeds the action twice: +Each class writes its own report, so one script merges them and derives the allocation series; the job then +feeds the action twice: ``` - Core / Avalonia / WPF *-report-full-compressed.json (one per project) + Core / Avalonia / WPF *-report-full.json (one per class) | v scripts/convert-benchmark-results.ps1 - +--> merged.json (the three Benchmarks arrays, the first report's host info) + +--> times.json (every report's Benchmarks array, the first report's host info) | --> github-action-benchmark tool: benchmarkdotnet "Time" | alert-threshold 150%, fail-on-alert false, summary-always true | @@ -274,13 +356,19 @@ three and derives the allocation series; the job then feeds the action twice: fail-on-alert: true, false while the PR has label accept-allocation-increase ``` -Benchmark names include their class's namespace, which differs per project, so names from the three reports -cannot collide. +Benchmark names include their class and its namespace, which differs per project, so names from different +reports cannot collide. - **Pull request**: `auto-push: false`; the action compares against the latest entry on the data branch and writes the result to the job summary. `comment-on-alert` is on only when the pull request comes from this repository, because a fork's token cannot comment. -- **Push to `main`**: `auto-push: true` appends the entry to the `benchmark-data` branch. +- **Push to `main`**: both steps run with `auto-push: false` and commit to the local data branch (the second + with `skip-fetch-gh-pages: true`, because fetching the branch again would be refused as a non-fast-forward); + the job then squashes the two commits, adds `latest/` and pushes one commit. The allocation step fails only + on a pull request: on `main` the increase was reviewed, and the new values become the baseline. +- **No data branch yet**: the first push to `main` creates `benchmark-data` with one README commit through the + GitHub API, because the action requires the branch to exist; a pull request before that skips the + comparison with a notice. - An allocation baseline of 0 bytes: the action computes the ratio against 0; the conversion script writes allocated bytes plus 1 so a 0 -> N change becomes an alert instead of a division by zero (spec: "a benchmark that allocated nothing on `main` SHALL fail on any allocation"). @@ -339,7 +427,7 @@ third-party action to detect changed paths (`git diff` does the same without a n The `benchmark-data` branch holds the action's history (`history/data.js` and its chart `index.html`) and, written by the job on `main` in the same push, the latest BenchmarkDotNet HTML and markdown reports under -`latest/`. `deploy-docs` builds the Pages content from two sources: +`latest/`, one per class, with an `index.html` that links them. `deploy-docs` builds the Pages content from two sources: ``` package-and-docs --(artifact: docs-site)--+ @@ -392,8 +480,11 @@ Alternative: commit result markdown into `docs/` from CI (a bot commit on `main` ## Migration Plan -1. Merge the projects, the job and the docs page. +1. Open the pull request; once its run has passed, archive this change in the same branch, then merge the + projects, the job and the docs page. 2. The first push to `main` creates the `benchmark-data` branch and the first entry. -3. Pull requests compare against it from then on. +3. Pull requests compare against it from then on. What only `main` can show - the data branch, the published + pages, a comparison - is a checklist in the pull request's description, checked after merge, because the + change is archived by then. Rollback: remove the job and restore the Pages artifact upload in `package-and-docs`; the branch can stay. diff --git a/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md index 12ceb85..406703a 100644 --- a/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md +++ b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md @@ -34,13 +34,19 @@ in the same culture. ### Requirement: Search-all lookups are benchmarked by the number of registered resource managers The suite SHALL measure a search-all lookup whose key exists only in the last registered resource manager, -for 1, 5 and 20 registered resource managers. +for 1, 5 and 20 registered resource managers, against hand-written code that searches the same resource +managers, in the same order and culture, until one contains the key. #### Scenario: Lookup cost grows with the registered managers - **WHEN** the search-all benchmark runs - **THEN** the results contain one row per manager count (1, 5 and 20), each finding the key in the last registered manager +#### Scenario: The baseline searches the same managers +- **WHEN** the search-all benchmark runs with a manager count +- **THEN** its baseline row searches the same resource manager instances in the same order and culture, so + the ratio shows only what ResXLocalization adds to the search + #### Scenario: Managers without the key are searched in the lookup culture - **WHEN** the search-all benchmark looks up a key in a specific culture, such as `de-DE` - **THEN** each registered manager that does not contain the key is searched in that culture, including its diff --git a/openspec/changes/add-benchmarks/tasks.md b/openspec/changes/add-benchmarks/tasks.md index 93cf095..f8ea110 100644 --- a/openspec/changes/add-benchmarks/tasks.md +++ b/openspec/changes/add-benchmarks/tasks.md @@ -2,61 +2,76 @@ ## 1. Project setup and feasibility check -- [ ] 1.1 Add `BenchmarkDotNet` to `Directory.Packages.props` (current stable version) and verify `dotnet restore ResXLocalization.slnx` succeeds -- [ ] 1.2 Create `benchmarks/Directory.Build.props` (imports the root props, adds `NoWarn` `CS1574;CS1591` with the same comment as the samples, and compiles `benchmarks/Shared/*.cs` into every benchmark project) and the three console projects `benchmarks/ResXLocalization.Core.Benchmarks` (`$(CurrentTargetFramework)`, Core and MessageFormat), `benchmarks/ResXLocalization.Avalonia.Benchmarks` (`$(CurrentTargetFramework)`, Avalonia) and `benchmarks/ResXLocalization.WPF.Benchmarks` (`$(CurrentWindowsTargetFramework)`, `UseWPF`), each with the library, Core and generator `ProjectReference`s the samples use and a `BenchmarkSwitcher` `Program`; verify all three build in Release -- [ ] 1.3 Add the three projects to `ResXLocalization.slnx` under `/04_benchmarks/`, and Core and Avalonia (not WPF) to `ResXLocalization.NonWindows.slnf`; verify `dotnet build ResXLocalization.NonWindows.slnf -c Release` and `dotnet test ResXLocalization.slnx -c Release` still pass and `dotnet test` runs no benchmark -- [ ] 1.4 Feasibility check: one Avalonia headless benchmark and one WPF `[STAThread]` benchmark that create a compiled XAML view with a `{l:Localize}` `TextBlock`, make it a window's content (headless window in Avalonia, a real `Window` outside the visible screen area in WPF) and check its text, and switch the culture once on a view whose `TextBlock` binds `[key]` on a minimal `TranslationSource` with `Source={x:Static ...}` (in Avalonia with compiled bindings), in their two projects; verify both run end to end with `dotnet run -c Release -- --filter *Feasibility* --inProcess --warmupCount 3 --iterationCount 3 --iterationTime 100` from each project and report time and allocations, record each project's total run time and confirm it leaves a suite of the planned size well under 10 minutes (otherwise cut the suite in the design before going on), then delete the feasibility check classes +- [x] 1.1 Add `BenchmarkDotNet` to `Directory.Packages.props` (current stable version) and verify `dotnet restore ResXLocalization.slnx` succeeds +- [x] 1.2 Create `benchmarks/Directory.Build.props` (imports the root props, adds `NoWarn` `CS1574;CS1591` with the same comment as the samples, and compiles `benchmarks/Shared/*.cs` into every benchmark project) and the three console projects `benchmarks/ResXLocalization.Core.Benchmarks` (`$(CurrentTargetFramework)`, Core and MessageFormat), `benchmarks/ResXLocalization.Avalonia.Benchmarks` (`$(CurrentTargetFramework)`, Avalonia) and `benchmarks/ResXLocalization.WPF.Benchmarks` (`$(CurrentWindowsTargetFramework)`, `UseWPF`), each with the library, Core and generator `ProjectReference`s the samples use and a `BenchmarkSwitcher` `Program`; verify all three build in Release +- [x] 1.3 Add the three projects to `ResXLocalization.slnx` under `/04_benchmarks/`, and Core and Avalonia (not WPF) to `ResXLocalization.NonWindows.slnf`; verify `dotnet build ResXLocalization.NonWindows.slnf -c Release` and `dotnet test ResXLocalization.slnx -c Release` still pass and `dotnet test` runs no benchmark +- [x] 1.4 Feasibility check: one Avalonia headless benchmark and one WPF `[STAThread]` benchmark that create a compiled XAML view with a `{l:Localize}` `TextBlock`, make it a window's content (headless window in Avalonia, a real `Window` outside the visible screen area in WPF) and check its text, and switch the culture once on a view whose `TextBlock` binds `[key]` on a minimal `TranslationSource` with `Source={x:Static ...}` (in Avalonia with compiled bindings), in their two projects; verify both run end to end with `dotnet run -c Release -- --filter *Feasibility* --inProcess --warmupCount 3 --iterationCount 3 --iterationTime 100` from each project and report time and allocations, record each project's total run time and confirm it leaves a suite of the planned size well under 10 minutes (otherwise cut the suite in the design before going on), then delete the feasibility check classes + - Note: Result: both ran end to end in about 10 s each. Avalonia's compiled indexer binding ignores `"Item[]"`; its `TranslationSource` raises `"Item"`, WPF's raises `Binding.IndexerName` (`"Item[]"`). ## 2. Resource data and shared configuration -- [ ] 2.1 Write by hand, in each of the three projects, `Resources/BenchmarkStrings.resx`, `BenchmarkStrings.de.resx` (every key) and a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` emits, with the same project-file wiring as the samples: in Core about 10 keys (plain text, positional and named arguments, one ICU message with two named arguments, enum values) plus one `Filler.resx` with a single other key; in Avalonia and WPF 10 keys including an argument key and the enum keys; verify the generator emits `BenchmarkStringsKeys`, the designer properties compile and the satellites build -- [ ] 2.2 Add the shared `[Config]` to `benchmarks/Shared/` (MemoryDiagnoser; JSON full, GitHub markdown and HTML exporters; one job with `InProcessEmitToolchain`, 3 warmup and 3 measured iterations of 100 ms; `ArtifactsPath` set to `artifacts/benchmarks//` from the repository root) and `[STAThread]` on the WPF project's `Main`, plus the culture setup that sets `CultureInfo.CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture; verify a run of a class in each project writes all three report files under `artifacts/benchmarks//results` and no `BenchmarkDotNet.Artifacts` folder, runs in-process (no generated project is built) and reports 3 warmup and 3 measured iterations -- [ ] 2.3 Add the self-check helper to `benchmarks/Shared/` that `[GlobalSetup]` calls to run each benchmark method once and throw on an unexpected text; verify by temporarily breaking one key that the class fails with an error naming the benchmark and produces no result row +- [x] 2.1 Write by hand, once in `benchmarks/Shared/Resources/`, `BenchmarkStrings.resx`, `BenchmarkStrings.de.resx` (every key) and a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` emits, linked into all three projects by `benchmarks/Directory.Build.props` with the designer's base name as `LogicalName`: plain texts, a text with two arguments, one ICU message with two named arguments and the 10 `Enum_Priority_Priority1`-`10` keys; plus, in Core only, one `Filler.resx` with a single other key; verify the generator emits `BenchmarkStringsKeys` in the designer's namespace, the designer properties compile, and every project embeds `RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings.resources` and builds a `de` satellite with the `.de` resources +- [x] 2.2 Add the shared `[Config]` to `benchmarks/Shared/` (MemoryDiagnoser; JSON full, GitHub markdown and HTML exporters; one job with `InProcessEmitToolchain`, 3 warmup and 15 measured iterations of 100 ms; `ArtifactsPath` set to `artifacts/benchmarks//` from the repository root) and `[STAThread]` on the WPF project's `Main`, plus the culture setup that sets `CultureInfo.CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture; verify a run of a class in each project writes all three report files under `artifacts/benchmarks//results` and no `BenchmarkDotNet.Artifacts` folder, runs in-process (no generated project is built) and reports 3 warmup and 15 measured iterations + - Note: The config adds only the JSON exporter; the GitHub markdown and HTML exporters come from BenchmarkDotNet's default configuration, which every `[Config]` is merged with. `Main` runs every benchmark when it gets no arguments, and returns 1 when nothing ran. 3 measured iterations were first planned; with one operation per iteration in the UI classes the error was often larger than the mean, so the job has 15 and the scaling classes 5. + - Note: `Priority`, `ArgumentValues`, `TranslationSource`, `ExpectedViewTexts` and `BenchmarkCulture` live in `benchmarks/Shared/`; `BenchmarkStrings` is one resource file set in `benchmarks/Shared/Resources/`, linked into every project by `benchmarks/Directory.Build.props` with its designer's base name as `LogicalName`; its designer and `BenchmarkStringsKeys` share the namespace of the shared code, so no project has a global using. `BenchmarkCulture.Use` sets `CurrentUICulture`, `Localizer.Current` and `TranslationSource.Instance` together. +- [x] 2.3 Add the self-check helper to `benchmarks/Shared/` that `[GlobalSetup]` calls to run each benchmark method once and throw on an unexpected text, with the expected texts read from the resx files (German from the `.de` satellite alone, without fallback); verify by temporarily breaking one key that the class fails with an error naming the benchmark and produces no result row ## 3. Core and MessageFormat benchmarks -- [ ] 3.1 `CodeLookupBenchmarks`: designer property baseline (`BenchmarkStrings.`), typed key, scoped key, search-all key and enum `Get`, in `de-DE`; verify with `--filter *CodeLookup*` that every row has a Ratio and Allocated value -- [ ] 3.2 `SearchAllBenchmarks` with `[Params(1, 5, 20)]` registered managers: 0, 4 or 19 separate `ResourceManager` instances over `Filler.resx`, then `BenchmarkStrings`, with the key only in `BenchmarkStrings`; verify three rows with rising time -- [ ] 3.3 `ArgumentBenchmarks`: `string.Format(culture, BenchmarkStrings., ...)` baseline, `FromValues`, `FromTuples`, `FromObject`; verify four rows and that the self-check passes for each -- [ ] 3.4 `MessageFormatBenchmarks`: one ICU message with two named arguments (a plural with `{Count}` and `{Name}`); a baseline that formats the designer property with one `new MessageFormatter()` and the same values and culture, against `Get` with `MessageArguments.FromTuples` and `UseMessageFormat()` installed; verify two rows and that the self-check confirms both produce the same text +- [x] 3.1 `CodeLookupBenchmarks`: designer property baseline (`ResXDesignerProperty`), `GetTypedKey`, `GetStringKeyScoped`, `GetStringKeySearchAll` and `GetEnum`, in `de-DE`; verify with `--filter *CodeLookup*` that every row has a Ratio and Allocated value +- [x] 3.2 `SearchAllBenchmarks` with `[Params(1, 5, 20)]` registered managers: 0, 4 or 19 separate `ResourceManager` instances over `Filler.resx`, then `BenchmarkStrings`, with the key only in `BenchmarkStrings`, against `HandWrittenSearchAll`, a loop of `GetString(key, culture)` over the same managers in the same order; verify three library rows with rising time and a baseline row for each + - Note: Six rows, not three: each manager count has its baseline row. The baseline was first the designer property, which knows the file: against it, 20 managers measured 65 times the time, nearly all `ResourceManager` misses that any search pays. Against the hand-written search the ratio is 1.03-1.14 and the allocations are identical (about 88 B per missed manager, from `ResourceManager`). +- [x] 3.3 `ArgumentBenchmarks`: `string.Format(culture, BenchmarkStrings., ...)` baseline (`StringFormatDesignerProperty`), `GetWithFromValues`, `GetWithFromTuples`, `GetWithFromObject`; verify four rows and that the self-check passes for each +- [x] 3.4 `MessageFormatBenchmarks`: one ICU message with two named arguments (a plural with `{Count}` and `{Name}`); a baseline that formats the designer property with one `new MessageFormatter()` and the same values and culture, against `Get` with `MessageArguments.FromTuples` and `UseMessageFormat()` installed (`MessageFormatterDirect`, `GetWithMessageFormat`); verify two rows and that the self-check confirms both produce the same text ## 4. Avalonia benchmarks -- [ ] 4.1 Headless session helper in the Avalonia project, started once in each class's `[GlobalSetup]` and running each body on its dispatcher; verify two classes in one run share the session without errors -- [ ] 4.2 Four 10-element `.axaml` views, each using one form for all elements and the 10 keys once (`{x:Static res:BenchmarkStrings.}`, `{l:Localize}` typed key, `{l:Localize}` with two `` children bound to the view's `DataContext`, `{l:LocalizeEnum}`), and `AvaloniaFormsBenchmarks`, one row per view (the `{x:Static}` view is the baseline), each opening 10 instances (100 elements) in a `StackPanel` as the headless window's content; verify four rows and that every element's self-check sees its expected text -- [ ] 4.3 Avalonia `TranslationSource`: singleton with a string indexer over `BenchmarkStrings.ResourceManager` and a `Culture` setter that raises the indexer's change notification; verify a compiled-XAML `TextBlock` bound to `[key]` on it updates after a culture change -- [ ] 4.4 Write the 10-element `TranslationSource` `.axaml` view (the 10 keys once) and add `AvaloniaScalingBenchmarks`, reusing the `{l:Localize}` and `{x:Static}` views of 4.2, with `[Params(100, 10000)]`: open a view (N / 10 views in a `StackPanel` as the window's content; baseline the `{x:Static}` views) and culture switch (the views built in `[IterationSetup]`, `InvocationCount = 1`, done when the last element shows the new culture's text; baseline the `TranslationSource` views); verify each operation has a baseline and a library row per N, and the run time of the class +- [x] 4.1 Headless session helper in the Avalonia project, started once in each class's `[GlobalSetup]` and running each body on its dispatcher; verify two classes in one run share the session without errors +- [x] 4.2 Four 10-element `.axaml` views under `Views/`, each using one form for all elements and the 10 keys once (`{x:Static res:BenchmarkStrings.}`, `{l:Localize}` typed key, `{l:Localize}` with two `` children bound to the view's `DataContext`, `{l:LocalizeEnum}`), and `AvaloniaFormsBenchmarks`, one row per view (the `{x:Static}` view is the baseline), each opening 10 instances (100 elements) in a `StackPanel` as the headless window's content; verify four rows and that every element's self-check sees its expected text + - Note: The enum view's 10 elements use the 10 members of the shared `Priority` enum. The forms class has `[WarmupCount(50)]` and the host runs without the tiered JIT's call-counting delay (`benchmarks/Directory.Build.props`), because with fewer warmups or the delay the first row ran partly unoptimized code and measured up to three times as slow as the same operation later in the process. +- [x] 4.3 `TranslationSource` in `benchmarks/Shared/`: singleton with a string indexer over `BenchmarkStrings.ResourceManager` and a `Culture` setter that raises one `PropertyChanged` with the framework's indexer name (`#if WINDOWS`: `"Item[]"` in WPF, `"Item"` in Avalonia); verify a compiled-XAML `TextBlock` bound to `[key]` on it updates after a culture change +- [x] 4.4 Write the 10-element `TranslationSource` `.axaml` view (the 10 keys once) and add `AvaloniaScalingBenchmarks`, reusing the `{l:Localize}` and `{x:Static}` views of 4.2, with `[Params(100, 10000)]`: open a view (N / 10 views in a `StackPanel` as the window's content; baseline the `{x:Static}` views) and culture switch (the views built in `[IterationSetup]`, `InvocationCount = 1`, done when the last element shows the new culture's text; baseline the `TranslationSource` views); verify each operation has a baseline and a library row per N, and the run time of the class + - Note: Every UI class has an `[IterationCleanup]` that empties the window, collects garbage and lets the dispatcher go idle; Avalonia processes pending work only down to `DispatcherPriority.Normal`, above layout, as WPF stops at `DataBind`. The application has the Fluent theme, for parity with WPF's default theme styles, and the culture is set on the BenchmarkDotNet thread because the headless dispatch flows its execution context. ## 5. WPF benchmarks -- [ ] 5.1 WPF helpers: a `Window` shown with `ShowActivated = false` outside the visible screen area that hosts the measured view, and a dispatcher flush with `DispatcherFrame`; verify a compiled-XAML view with a `{l:Localize}` `TextBlock` shows its text after one flush -- [ ] 5.2 WPF `TranslationSource`, mirroring 4.3; verify the same way -- [ ] 5.3 The four 10-element `.xaml` views and `WpfFormsBenchmarks`, mirroring 4.2, all `[STAThread]`; verify the same rows as Avalonia and that the self-checks pass -- [ ] 5.4 The `TranslationSource` `.xaml` view and `WpfScalingBenchmarks`, mirroring 4.4; verify the same rows as Avalonia, and that the whole WPF project runs locally in under 6 minutes -- [ ] 5.5 Run the `ui_parity_reviewer` over the Avalonia and WPF benchmark classes and resolve any gap it reports +- [x] 5.1 WPF helpers: a `Window` shown with `ShowActivated = false` outside the visible screen area that hosts the measured view, and a dispatcher flush with `DispatcherFrame`; verify a compiled-XAML view with a `{l:Localize}` `TextBlock` shows its text after one flush +- [x] 5.2 Use the shared `TranslationSource` in WPF; verify the same way as 4.3 +- [x] 5.3 The four 10-element `.xaml` views under `Views/` and `WpfFormsBenchmarks`, mirroring 4.2, all `[STAThread]`; verify the same rows as Avalonia and that the self-checks pass + - Note: Mirrors 4.2, but with `[WarmupCount(300)]`: WPF's ReadyToRun code takes a few hundred one-operation iterations to be recompiled to optimized code, and with 50 the first row of the process measured about 2 ms too slow. +- [x] 5.4 The `TranslationSource` `.xaml` view and `WpfScalingBenchmarks`, mirroring 4.4; verify the same rows as Avalonia, and that the whole WPF project runs locally in under 6 minutes + - Note: Mirrors 4.4. The idle cleanup matters most here: without it WPF kept the weak-event listeners of every earlier view, and the 100-element culture switch took about six times as long. The whole WPF project runs in about 35 s. +- [x] 5.5 Run the `ui_parity_reviewer` over the Avalonia and WPF benchmark classes and resolve any gap it reports + - Note: The reviewer found one gap, Avalonia's culture switch running all dispatcher work including layout; fixed. ## 6. CI integration -- [ ] 6.1 Add `scripts/convert-benchmark-results.ps1` that reads the three projects' BenchmarkDotNet full JSON reports, writes `merged.json` (their `Benchmarks` arrays, the first report's host info) and `allocations.json` (`customSmallerIsBetter`, allocated bytes + 1 per benchmark); verify on local results that every benchmark of all three projects appears once in each file, with a number -- [ ] 6.2 Add the `benchmark-changes` job to `.github/workflows/ci.yml`: `git diff --name-only` against the pull request's base or the push's previous commit, with the output `run` true when anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`; verify on a documentation-only test pull request that `run` is false and the benchmark jobs are skipped, and on a code change that it is true -- [ ] 6.3 Add the `benchmarks` matrix job (Core, Avalonia, WPF) on `windows-latest`, running only when `benchmark-changes` says so: setup, restore, build and run its project with `--join`, upload its `artifacts/benchmarks//` folder, `timeout-minutes: 15`; verify with `pwsh -File scripts/tidy-code.ps1 -Scope all -Check` and `scripts/verify-line-endings.ps1` -- [ ] 6.4 Add the `benchmark-report` job that needs the matrix: download the three reports, merge with `scripts/convert-benchmark-results.ps1`, run `benchmark-action/github-action-benchmark` pinned by SHA twice (`benchmarkdotnet` "Time" at 150% without failing, `customSmallerIsBetter` "Allocations" at 125% failing, `summary-always`, `comment-on-alert` only for same-repository pull requests, `auto-push` only on `main`, branch `benchmark-data`, `max-items-in-chart: 200`), with job-level `contents: write` and `pull-requests: write`; verify with the lint job -- [ ] 6.5 Make the "Allocations" step's `fail-on-alert` false while the pull request has the label `accept-allocation-increase`, and add `labeled` and `unlabeled` to the `pull_request` trigger's `types` (with `opened`, `synchronize`, `reopened`); verify with the lint job, and on a test pull request that adding and removing the label re-runs the check and turns a failing allocation alert into a passing one and back -- [ ] 6.6 Create the `accept-allocation-increase` label in the repository (maintainer step, with a description saying it accepts a reviewed allocation increase for one pull request); verify it appears in the repository's labels -- [ ] 6.7 On `main`, have `benchmark-report` copy the HTML and markdown reports into `latest/` on the `benchmark-data` branch; verify on a pull-request run that the comparison is reported within 10 minutes of the benchmark jobs starting and its summary shows both series +- [x] 6.1 Add `scripts/convert-benchmark-results.ps1` that reads every class's BenchmarkDotNet full JSON report, writes `times.json` (their `Benchmarks` arrays, the first report's host info) and `allocations.json` (`customSmallerIsBetter`, allocated bytes + 1 per benchmark); verify on local results that every benchmark of all three projects appears once in each file, with a number +- [x] 6.2 Add the `benchmark-changes` job to `.github/workflows/ci.yml`: `git diff --name-only` against the pull request's base or the push's previous commit, with the output `run` true when anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`; verify with the lint job +- [x] 6.3 Add the `benchmarks` matrix job (Core, Avalonia, WPF) on `windows-latest`, running only when `benchmark-changes` says so: setup, restore, build and run its project (one report per class, no `--join`), upload its `artifacts/benchmarks//` folder, `timeout-minutes: 15`; verify with `pwsh -File scripts/tidy-code.ps1 -Scope all -Check` and `scripts/verify-line-endings.ps1` +- [x] 6.4 Add the `benchmark-report` job that needs the matrix: download the three reports, merge with `scripts/convert-benchmark-results.ps1`, run `benchmark-action/github-action-benchmark` pinned by SHA twice (`benchmarkdotnet` "Time" at 150% without failing, `customSmallerIsBetter` "Allocations" at 125% failing, `summary-always`, `comment-on-alert` only for same-repository pull requests, `auto-push` only on `main`, branch `benchmark-data`, `max-items-in-chart: 200`), with job-level `contents: write` and `pull-requests: write`; verify with the lint job + - Note: Both action steps run with `auto-push: false`; the second skips the fetch. On `main` a final step squashes their commits, adds `latest/` and pushes once, and creates the data branch through the API on the first run. The allocation step fails only on pull requests. +- [x] 6.7 On `main`, have `benchmark-report` copy every class's HTML and markdown report into `latest/` on the `benchmark-data` branch, with an `index.html` that links them ## 7. Publishing and documentation -- [ ] 7.1 Change `package-and-docs` to upload the DocFX site as an ordinary artifact, and `deploy-docs` to need `package-and-docs` and `benchmark-report` with `if: always() && needs.package-and-docs.result == 'success'`, download the site, check out `benchmark-data` into `benchmarks/` inside it, then upload and deploy the Pages artifact; verify the workflow passes the lint job -- [ ] 7.2 Write a short `docs/guides/benchmarks.md` (a table of each benchmark and its baseline, one sentence on the Ratio and Allocated columns, absolute links to `benchmarks/latest/` and `benchmarks/history/`) and add it to `docs/toc.yml` under Guides; verify `dotnet tool run docfx build build/docfx/docfx.json --warningsAsErrors` passes -- [ ] 7.3 Add a "Running the benchmarks" section to `CONTRIBUTING.md` (local command, `--filter`, how CI gates allocations and alerts on time, and when a maintainer applies `accept-allocation-increase`) and the `benchmarks/` row to the project map in `AGENTS.md`; verify the documented command runs as written -- [ ] 7.4 Add an `[Unreleased]` entry in `CHANGELOG.md` for the published benchmark results page; verify the changelog has no dated section added +- [x] 7.1 Change `package-and-docs` to upload the DocFX site as an ordinary artifact, and `deploy-docs` to need `package-and-docs` and `benchmark-report` with `if: always() && needs.package-and-docs.result == 'success'`, download the site, check out `benchmark-data` into `benchmarks/` inside it, then upload and deploy the Pages artifact; verify the workflow passes the lint job +- [x] 7.2 Write a short `docs/guides/benchmarks.md` (a table of each benchmark and its baseline, one sentence on the Ratio and Allocated columns, absolute links to `benchmarks/latest/` and `benchmarks/history/`) and add it to `docs/toc.yml` under Guides; verify `dotnet tool run docfx build build/docfx/docfx.json --warningsAsErrors` passes +- [x] 7.3 Add a "Running the benchmarks" section to `CONTRIBUTING.md` (local command, `--filter`, how CI gates allocations and alerts on time, and when a maintainer applies `accept-allocation-increase`) and the `benchmarks/` row to the project map in `AGENTS.md`; verify the documented command runs as written +- [x] 7.4 Add an `[Unreleased]` entry in `CHANGELOG.md` for the published benchmark results page; verify the changelog has no dated section added ## 8. Integration check -- [ ] 8.1 Run `pwsh -File scripts/pre-commit-gate.ps1` and verify it passes with the benchmark projects included -- [ ] 8.2 After merge, verify the first push to `main` creates the `benchmark-data` branch and the deployed site serves `benchmarks/latest/` and `benchmarks/history/`, and that the next pull request's summary compares against that entry +- [x] 8.1 Run `pwsh -File scripts/pre-commit-gate.ps1` and verify it passes with the benchmark projects included + +## Manual steps and checks + +Done by a maintainer on GitHub, outside this change's tasks: they need the pull request, its CI run or `main`. + +- Create the `accept-allocation-increase` label in the repository, with a description saying it accepts a reviewed allocation increase for one pull request. +- On this pull request's run: `benchmark-changes` outputs `run=true`; the benchmark jobs and the report job pass within 10 minutes of the benchmark jobs starting; the report job shows the "No benchmark baseline" notice; the results are attached to the run. Locally, the job's `git diff` command against a documentation-only commit gives `run=false`. +- Add a post-merge checklist to the pull request's description: the first push to `main` creates the `benchmark-data` branch, the deployed site serves `benchmarks/latest/` and `benchmarks/history/`, and the next pull request's summary compares against that entry with both series. ## Workflow follow-up -- Archive the change after the integration check on `main` has passed. +- Archive the change in this branch after the pull request's run has passed and before it is merged; the post-merge checklist lives in the pull request, not here. diff --git a/scripts/convert-benchmark-results.ps1 b/scripts/convert-benchmark-results.ps1 new file mode 100644 index 0000000..76d86e4 --- /dev/null +++ b/scripts/convert-benchmark-results.ps1 @@ -0,0 +1,88 @@ +<# +.SYNOPSIS + Merges the BenchmarkDotNet reports of the benchmark projects into the two files the CI benchmark comparison + reads: times.json (mean time) and allocations.json (allocated bytes). + +.DESCRIPTION + benchmark-action/github-action-benchmark reads one BenchmarkDotNet JSON file and compares only the mean + time. Every benchmark class writes its own full JSON report, so this script + + - merges every *-report-full.json below -ReportDirectory into times.json: their Benchmarks arrays one + after the other, and the host information of the first report, for the "Time" series (tool: + benchmarkdotnet); + - writes allocations.json in the action's customSmallerIsBetter format, one entry per benchmark with its + allocated bytes per operation, for the "Allocations" series. + + An allocation is written as its bytes plus 1. The action computes a ratio to the value on main, and a + benchmark that allocated nothing there would otherwise be a division by zero instead of an alert when it + starts to allocate. + + A benchmark name is BenchmarkDotNet's FullName, which includes its class and the namespace of its project, + so the names of the classes and projects cannot collide; the script fails if two benchmarks share one anyway, or if a benchmark has + no result, because a missing row would silently drop out of the comparison. + +.PARAMETER ReportDirectory + The directory searched, recursively, for the BenchmarkDotNet *-report-full.json files. + +.PARAMETER OutputDirectory + The directory times.json and allocations.json are written to. It is created if it does not exist. + +.EXAMPLE + pwsh -File scripts/convert-benchmark-results.ps1 -ReportDirectory artifacts/benchmarks -OutputDirectory artifacts/benchmarks/series +#> +#requires -Version 7.0 +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $ReportDirectory, + + [Parameter(Mandatory = $true)] + [string] $OutputDirectory +) + +$ErrorActionPreference = 'Stop' + +$reportFiles = @( + Get-ChildItem -LiteralPath $ReportDirectory -Recurse -File -Filter '*-report-full.json' | Sort-Object FullName +) + +if ($reportFiles.Count -eq 0) +{ + throw "No BenchmarkDotNet *-report-full.json file was found below $ReportDirectory." +} + +$reports = @($reportFiles | ForEach-Object { Get-Content -LiteralPath $_.FullName -Raw | ConvertFrom-Json }) +$benchmarks = @($reports | ForEach-Object { $_.Benchmarks }) +$allocations = New-Object System.Collections.Generic.List[Object] +$names = New-Object System.Collections.Generic.HashSet[String] + +foreach ($benchmark in $benchmarks) +{ + if (-not $names.Add($benchmark.FullName)) + { + throw "Two benchmarks are named $($benchmark.FullName)." + } + + if ($null -eq $benchmark.Statistics -or $null -eq $benchmark.Memory) + { + throw "$($benchmark.FullName) has no result. Its run failed; see the BenchmarkDotNet log." + } + + $allocations.Add( + [ordered]@{ + name = $benchmark.FullName + unit = 'bytes' + value = [long] $benchmark.Memory.BytesAllocatedPerOperation + 1 + } + ) +} + +$times = $reports[0] | Select-Object -Property * -ExcludeProperty Benchmarks +$times | Add-Member -NotePropertyName Benchmarks -NotePropertyValue $benchmarks + +New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null +$times | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'times.json') -Encoding utf8NoBOM +ConvertTo-Json -InputObject $allocations -Depth 4 | + Set-Content -LiteralPath (Join-Path $OutputDirectory 'allocations.json') -Encoding utf8NoBOM + +Write-Output "Merged $($benchmarks.Count) benchmark(s) from $($reportFiles.Count) report(s) into $OutputDirectory."