diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b8898d..ad8274a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,15 @@ on: branches: - main - release/** + # labeled and unlabeled re-run the checks when a maintainer adds or removes accept-allocation-increase, + # which turns a failing allocation alert of the benchmark comparison into a passing one and back. They + # re-run the whole workflow; the concurrency group below cancels the run in progress, and labeling is rare. + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled push: branches: - main @@ -25,6 +34,8 @@ env: AOT_OUTPUT: artifacts/package-aot AVALONIA_CONSUMER_PROJECT: tests/package-consumption/AvaloniaConsumer/AvaloniaConsumer.csproj COMBINED_CONSUMER_PROJECT: tests/package-consumption/CombinedConsumer/CombinedConsumer.csproj + BENCHMARK_DATA_BRANCH: benchmark-data + BENCHMARK_REPORTS: artifacts/benchmarks CONFIGURATION: Release CONSUMER_DIRECTORY: tests/package-consumption DOCFX_CONFIG: build/docfx/docfx.json @@ -332,12 +343,16 @@ jobs: echo "version=${version}" >> "${GITHUB_OUTPUT}" echo "Packed version ${version}" + # An ordinary artifact rather than the Pages artifact: deploy-docs adds the benchmark results to the + # site before it builds the Pages artifact. Only deploy-docs (pushes to main) consumes it. - name: Upload documentation site - # The Pages artifact is only consumed by deploy-docs (pushes to main); skip the upload on PRs. if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + name: docs-site path: ${{ env.DOCS_SITE }} + if-no-files-found: error + retention-days: 1 - name: Upload NuGet packages uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -350,6 +365,239 @@ jobs: retention-days: 30 compression-level: 0 + # Decides whether the benchmarks run: only when something they measure, or how they are run, changed, so a + # documentation-only pull request is not held up by them. A skipped job counts as passed for a required + # check. Without a previous commit to compare with - a new branch, a tag, a manual run - they run. + benchmark-changes: + name: Detect benchmark changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run: ${{ steps.changes.outputs.run }} + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Compare the changed files + id: changes + shell: bash + env: + BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + run: | + if [ -z "${BASE}" ] || ! git cat-file -e "${BASE}^{commit}" 2> /dev/null; then + echo "No previous commit to compare with; the benchmarks run." + echo "run=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + pattern='^(src/|benchmarks/|build/|Directory\.[^/]+$|global\.json$|\.github/workflows/ci\.yml$|scripts/convert-benchmark-results\.ps1$)' + + if git diff --name-only "${BASE}" HEAD | grep -qE "${pattern}"; then + echo "run=true" >> "${GITHUB_OUTPUT}" + else + echo "Nothing the benchmarks measure changed; they are skipped." + echo "run=false" >> "${GITHUB_OUTPUT}" + fi + + # One suite per entry, in parallel with the build jobs, all on one runner type so that the history stays + # comparable. Every benchmark class writes its own report, so each table's Ratio column is against that + # class's baseline; --join would merge a suite's classes into one table with one baseline. The suites check + # their own results before measuring and exit non-zero when a benchmark fails or none ran, so a broken + # benchmark fails here instead of missing from the report. + benchmarks: + name: Benchmarks (${{ matrix.suite }}) + runs-on: windows-latest + timeout-minutes: 15 + needs: benchmark-changes + if: needs.benchmark-changes.outputs.run == 'true' + strategy: + fail-fast: false + matrix: + suite: + - Core + - Avalonia + - WPF + env: + BENCHMARK_PROJECT: benchmarks/ResXLocalization.${{ matrix.suite }}.Benchmarks/ResXLocalization.${{ matrix.suite }}.Benchmarks.csproj + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: 10.0.x + dotnet-quality: ga + + - name: Cache NuGet packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ env.NUGET_PACKAGES }} + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets', '**/*.slnx', '.config/dotnet-tools.json') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Restore + run: dotnet restore ${{ env.BENCHMARK_PROJECT }} --configfile ${{ env.NUGET_CONFIG }} + + - name: Build + run: dotnet build ${{ env.BENCHMARK_PROJECT }} --configuration ${{ env.CONFIGURATION }} --no-restore /p:ContinuousIntegrationBuild=true + + - name: Run benchmarks + run: dotnet run --project ${{ env.BENCHMARK_PROJECT }} --configuration ${{ env.CONFIGURATION }} --no-build -- --filter "*" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: benchmark-results-${{ matrix.suite }} + path: ${{ env.BENCHMARK_REPORTS }}/ResXLocalization.${{ matrix.suite }}.Benchmarks/ + if-no-files-found: error + retention-days: 30 + + # Compares the results with the latest results from main and reports both series in the job summary: a time + # above 150% is an alert, an allocation above 125% fails a pull request - unless a maintainer accepted the + # increase with the label accept-allocation-increase. On main the results are appended to the history on the + # benchmark-data branch, with this run's reports under latest/, which deploy-docs publishes. Only this job + # can write. + benchmark-report: + name: Benchmark report + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: benchmarks + permissions: + contents: write + pull-requests: write + env: + BENCHMARK_SERIES: artifacts/benchmarks/series + # A fork's token cannot comment, so only a pull request from this repository comments on an alert; the + # job summary has the comparison either way. + COMMENT_ON_ALERT: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + IS_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Download benchmark results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: benchmark-results-* + path: ${{ env.BENCHMARK_REPORTS }} + + - name: Merge benchmark results + run: pwsh -NoProfile -NonInteractive -File scripts/convert-benchmark-results.ps1 -ReportDirectory ${{ env.BENCHMARK_REPORTS }} -OutputDirectory ${{ env.BENCHMARK_SERIES }} + + # The action needs the data branch to exist. The first push to main creates it, with one commit and no + # results; a pull request before that has nothing to compare with. + - name: Find the benchmark data branch + id: data-branch + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + if git ls-remote --exit-code --heads origin "${BENCHMARK_DATA_BRANCH}" > /dev/null; then + echo "exists=true" >> "${GITHUB_OUTPUT}" + elif [ "${IS_MAIN}" = "true" ]; then + tree="$(gh api "repos/${GITHUB_REPOSITORY}/git/trees" -f 'tree[][path]=README.md' -f 'tree[][mode]=100644' -f 'tree[][type]=blob' -f 'tree[][content]=Benchmark results of main, written by CI. This branch holds no code and is never merged.' --jq .sha)" + commit="$(gh api "repos/${GITHUB_REPOSITORY}/git/commits" -f message="Create the benchmark data branch" -f tree="${tree}" --jq .sha)" + gh api "repos/${GITHUB_REPOSITORY}/git/refs" -f ref="refs/heads/${BENCHMARK_DATA_BRANCH}" -f sha="${commit}" > /dev/null + echo "exists=true" >> "${GITHUB_OUTPUT}" + else + echo "::notice title=No benchmark baseline::main has no benchmark results yet, so there is nothing to compare with." + echo "exists=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Compare times + if: steps.data-branch.outputs.exists == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: Time + tool: benchmarkdotnet + output-file-path: ${{ env.BENCHMARK_SERIES }}/times.json + gh-pages-branch: ${{ env.BENCHMARK_DATA_BRANCH }} + benchmark-data-dir-path: history + github-token: ${{ github.token }} + auto-push: false + alert-threshold: "150%" + fail-on-alert: false + comment-on-alert: ${{ env.COMMENT_ON_ALERT }} + summary-always: true + max-items-in-chart: 200 + + # skip-fetch-gh-pages: the step above fetched the branch and committed to it locally, and fetching it + # again would be refused as a non-fast-forward update. Only a pull request fails on an alert: on main the + # increase was reviewed, and the new values become the baseline. + - name: Compare allocations + if: steps.data-branch.outputs.exists == 'true' + uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 + with: + name: Allocations + tool: customSmallerIsBetter + output-file-path: ${{ env.BENCHMARK_SERIES }}/allocations.json + gh-pages-branch: ${{ env.BENCHMARK_DATA_BRANCH }} + benchmark-data-dir-path: history + github-token: ${{ github.token }} + auto-push: false + skip-fetch-gh-pages: true + alert-threshold: "125%" + fail-on-alert: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'accept-allocation-increase') }} + comment-on-alert: ${{ env.COMMENT_ON_ALERT }} + summary-always: true + max-items-in-chart: 200 + + # Each of the two steps above committed to the local data branch. Squash them, add this run's HTML and + # markdown reports under latest/ (one per benchmark class) with an index page, and push one commit. + - name: Publish to the benchmark data branch + if: env.IS_MAIN == 'true' && steps.data-branch.outputs.exists == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + reports="$(mktemp -d)" + links="" + for suite in Core Avalonia WPF; do + for html in "${BENCHMARK_REPORTS}/benchmark-results-${suite}/results"/*-report.html; do + report="$(basename "${html}" -report.html)" + cp "${html}" "${reports}/${report}.html" + cp "${html%.html}-github.md" "${reports}/${report}.md" + links="${links}
  • ${suite}: ${report##*.} (markdown)
  • " + done + done + + { + echo '' + echo '' + echo 'ResXLocalization benchmark results' + echo '' + echo '

    ResXLocalization benchmark results

    ' + echo "

    Commit ${GITHUB_SHA}. Each report starts with the hardware and runtime it was measured on.

    " + echo '' + echo '

    History of every benchmark on main

    ' + echo '' + echo '' + } > "${reports}/index.html" + + git switch "${BENCHMARK_DATA_BRANCH}" + git reset --soft HEAD~2 + rm -rf latest + mkdir latest + cp "${reports}"/* latest/ + git add history latest + git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" commit --quiet -m "Benchmark results for ${GITHUB_SHA}" + git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "${BENCHMARK_DATA_BRANCH}" + # Prove the source generator's Roslyn floor forever: consume the packed packages with ONLY the .NET 8 # SDK. A generator compiled against a newer Roslyn than the .NET 8 compiler would emit CS9057, skip # generation, and fail the consumer build on the missing …Keys types. @@ -549,14 +797,19 @@ jobs: retention-days: 14 # Publish the DocFX site to GitHub Pages on every push to main (requires Pages to be set to - # "GitHub Actions" as the source in the repository settings). + # "GitHub Actions" as the source in the repository settings), with the benchmark-data branch under + # benchmarks/. It waits for benchmark-report without depending on it: when the benchmarks fail or are + # skipped, the site is deployed with the last results the branch has. deploy-docs: name: Deploy docs to GitHub Pages runs-on: ubuntu-latest timeout-minutes: 10 - needs: package-and-docs - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - package-and-docs + - benchmark-report + if: always() && needs.package-and-docs.result == 'success' && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: + contents: read pages: write id-token: write environment: @@ -564,6 +817,40 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Download documentation site + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: docs-site + path: ${{ env.DOCS_SITE }} + + - name: Find the benchmark data branch + id: data-branch + shell: bash + run: | + if git ls-remote --exit-code --heads "https://github.com/${GITHUB_REPOSITORY}.git" "${BENCHMARK_DATA_BRANCH}" > /dev/null; then + echo "exists=true" >> "${GITHUB_OUTPUT}" + else + echo "exists=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Add the benchmark results + if: steps.data-branch.outputs.exists == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ env.BENCHMARK_DATA_BRANCH }} + path: ${{ env.DOCS_SITE }}/benchmarks + persist-credentials: false + + - name: Remove the checkout metadata + if: steps.data-branch.outputs.exists == 'true' + shell: bash + run: rm -rf "${DOCS_SITE}/benchmarks/.git" + + - name: Upload the Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ${{ env.DOCS_SITE }} + - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5 diff --git a/AGENTS.md b/AGENTS.md index 2a41825..907a0f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,7 @@ supports neither). | `src/ResXLocalization.WPF` | The same for WPF. Windows-only, no Native AOT | | `samples/` | Two runnable showcases, one per UI framework. Excluded from coverage | | `tests/` | Five suites: Core and `tests/ResXLocalization.MessageFormat.Tests` (both on both frameworks), the generator, and one sample-test suite per UI framework | +| `benchmarks/` | Three BenchmarkDotNet suites - Core with MessageFormat, Avalonia headless, WPF - each against the code an application writes without the library, and `Shared/`, compiled into all three. Not test projects; CI runs them and gates on allocations | | `tests/package-consumption/` | Package-only consumers. The repository's `Directory.Build.*` deliberately does **not** apply there | | `build/` | The shared resx and packaging targets, and the DocFX configuration | | `scripts/` | Every check and workflow, as PowerShell 7 | diff --git a/CHANGELOG.md b/CHANGELOG.md index 28cd0bf..63044cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- A BenchmarkDotNet suite under `benchmarks/` that measures code lookups, ICU MessageFormat and the Avalonia + and WPF markup extensions against the `.resx` designer class. +- A [benchmark results page](https://rent-a-developer.github.io/ResXLocalization/guides/benchmarks.html) in + the documentation: what each lookup, markup extension and culture switch costs compared with the `.resx` + designer class, with the latest results and their history on `main`. + ## [2.0.0] - 2026-10-20 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f00a0d..6d501e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,6 +179,36 @@ starts and answers wrongly. `scripts/verify-package-aot.ps1` publishes the packa The WPF package is exempt — WPF does not support Native AOT. +## Running the benchmarks + +`benchmarks/` holds three BenchmarkDotNet suites: `ResXLocalization.Core.Benchmarks` (Core and +MessageFormat), `ResXLocalization.Avalonia.Benchmarks` (headless) and `ResXLocalization.WPF.Benchmarks` +(Windows only). Each compares the library with the code an application writes without it, see +[`docs/guides/benchmarks.md`](docs/guides/benchmarks.md). Run one suite from the repository root, all of its +benchmarks or the classes `--filter` selects: + +```shell +dotnet run --project benchmarks/ResXLocalization.Core.Benchmarks -c Release +dotnet run --project benchmarks/ResXLocalization.Core.Benchmarks -c Release -- --filter "*CodeLookup*" +``` + +The reports land in `artifacts/benchmarks//results`. Every class checks once, before it measures, +that each benchmark produces the expected text, and stops with an error naming the benchmark when it does +not. + +CI runs the three suites on every pull request and every push to `main` that changes code, and compares +them with the latest results from `main`: + +- **More allocations fail the check**: a benchmark that allocates more than 125% of its bytes on `main`, or + anything at all where it allocated nothing. +- **A slower time is an alert, not a failure**: above 150% of the time on `main` it is listed in the job + summary and commented on the pull request. Shared CI machines vary too much in speed to fail on time. + +When an allocation increase is deliberate - a feature that has to allocate - a maintainer reviews it and adds +the label `accept-allocation-increase` to the pull request. The check then reports the increase and passes, +and the merged values become the baseline for every later pull request. Removing the label makes the check +fail again. + ## Public API The four runtime projects track their public surface with diff --git a/Directory.Packages.props b/Directory.Packages.props index 21248f4..b454bc1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -49,6 +49,10 @@ + + + + + + + + + Exe + + + + + $(NoWarn);CS1574;CS1591 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs new file mode 100644 index 0000000..932a4e8 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaFormsBenchmarks.cs @@ -0,0 +1,71 @@ +using Avalonia.Controls; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// Opening 100 text elements that all use one form - 10 instances of a 10-element view as a headless window's +/// content - against the same layout with {x:Static} of the designer class's properties. +/// +/// +/// 50 warmup iterations instead of the configuration's 3: an iteration here is one operation, and the tiered JIT +/// promotes a method only after 30 calls, so with fewer warmups the measured rows ran partly unoptimized code +/// and the first rows looked slower than the later ones (see also the call-counting delay in +/// benchmarks/Directory.Build.props). +/// +[Config(typeof(BenchmarkConfig))] +[WarmupCount(50)] +public class AvaloniaFormsBenchmarks +{ + private const int ViewCount = 10; + + private Window window = null!; + + [GlobalCleanup] + public void Cleanup() => AvaloniaSession.Run(this.window.Close); + + [IterationCleanup] + public void Idle() => AvaloniaSession.EmptyWindow(this.window); + + [Benchmark] + public StackPanel Localize() => this.Open(static () => new LocalizeView()); + + [Benchmark] + public StackPanel LocalizeEnum() => this.Open(static () => new EnumView()); + + [Benchmark] + public StackPanel LocalizeWithArguments() => this.Open(static () => new ArgumentsView()); + + [GlobalSetup] + public void Setup() + { + AvaloniaSession.Start(); + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = AvaloniaSession.Run(AvaloniaSession.ShowWindow); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Repeat( + name switch + { + nameof(LocalizeWithArguments) => ExpectedViewTexts.GermanWithArguments, + nameof(LocalizeEnum) => ExpectedViewTexts.GermanEnum, + _ => ExpectedViewTexts.German, + }, + ViewCount + ), + AvaloniaSession.GetTextsOfAllTextBlocks + ); + } + + [Benchmark(Baseline = true)] + public StackPanel Static() => this.Open(static () => new StaticView()); + + private StackPanel Open(Func createView) => + AvaloniaSession.Run(() => AvaloniaSession.Open(this.window, ViewCount, createView)); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs new file mode 100644 index 0000000..d69d3d6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaScalingBenchmarks.cs @@ -0,0 +1,124 @@ +using Avalonia.Controls; +using Avalonia.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// Opening a view with {l:Localize} elements, against {x:Static}, and +/// switching the culture while they are live, against the hand-written . The +/// elements are / 10 instances of a 10-element view in one headless window. +/// +/// +/// Every iteration is one operation (InvocationCount = 1), which the culture switch needs for its +/// [IterationSetup] and which keeps the 10,000-element rows short. 5 measured iterations instead of the +/// configuration's 15, because an iteration of a 10,000-element row takes up to a second plus its setup and +/// cleanup. +/// +[Config(typeof(BenchmarkConfig))] +[CategoriesColumn] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[InvocationCount(1)] +[IterationCount(5)] +public class AvaloniaScalingBenchmarks +{ + private const string OpenCategory = "Open"; + + private const string SwitchCategory = "Switch"; + + private Window window = null!; + + [Params(100, 10_000)] + public int ElementCount { get; set; } + + private int ViewCount => this.ElementCount / 10; + + [GlobalCleanup] + public void Cleanup() => AvaloniaSession.Run(this.window.Close); + + [IterationCleanup] + public void Idle() => AvaloniaSession.EmptyWindow(this.window); + + [Benchmark] + [BenchmarkCategory(OpenCategory)] + public StackPanel OpenLocalize() => this.Open(static () => new LocalizeView()); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(OpenCategory)] + public StackPanel OpenStatic() => this.Open(static () => new StaticView()); + + [GlobalSetup] + public void Setup() + { + AvaloniaSession.Start(); + Localizer.Current.ClearResourceManagers(); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = AvaloniaSession.Run(AvaloniaSession.ShowWindow); + + SelfCheck.Verify( + this, + name => + ExpectedViewTexts.Repeat( + name.StartsWith(SwitchCategory, StringComparison.Ordinal) + ? ExpectedViewTexts.English + : ExpectedViewTexts.German, + this.ViewCount + ), + AvaloniaSession.GetTextsOfAllTextBlocks + ); + } + + [IterationSetup(Targets = [nameof(SwitchLocalize)])] + public void SetupLocalizeViews() => this.SetupViews(static () => new LocalizeView()); + + [IterationSetup(Targets = [nameof(SwitchTranslationSource)])] + public void SetupTranslationSourceViews() => this.SetupViews(static () => new TranslationSourceView()); + + [Benchmark] + [BenchmarkCategory(SwitchCategory)] + public StackPanel SwitchLocalize() => + this.Switch(static () => Localizer.Current.CurrentCulture = BenchmarkCulture.English); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(SwitchCategory)] + public StackPanel SwitchTranslationSource() => + this.Switch(static () => TranslationSource.Instance.Culture = BenchmarkCulture.English); + + private StackPanel Open(Func createView) => + AvaloniaSession.Run(() => AvaloniaSession.Open(this.window, this.ViewCount, createView)); + + /// + /// Builds the views of a culture switch in German, outside the measurement. The previous iteration's views + /// are removed first, so that only these views are live when BenchmarkDotNet collects garbage before the + /// iteration. The culture is set on the UI thread, because views an earlier benchmark left may still be + /// subscribed and update their texts; the CurrentUICulture it sets there ends with the call, and none + /// of these views reads it. + /// + /// Creates one view. + private void SetupViews(Func createView) => + AvaloniaSession.Run(() => + { + this.window.Content = null; + BenchmarkCulture.Use(BenchmarkCulture.German); + _ = AvaloniaSession.Open(this.window, this.ViewCount, createView); + }); + + /// + /// Switches the culture and processes the dispatcher's pending binding work, down to + /// , which is above layout and rendering, as in WPF. Avalonia updates + /// a binding on the UI thread as soon as its source raises PropertyChanged, so every element shows the + /// new text when this returns; the self-check confirms it for all of them. + /// + /// Switches the culture. + /// The panel with the views. + private StackPanel Switch(Action switchCulture) => + AvaloniaSession.Run(() => + { + switchCulture(); + Dispatcher.UIThread.RunJobs(DispatcherPriority.Normal); + return (StackPanel)this.window.Content!; + }); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs new file mode 100644 index 0000000..c57e9e6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/AvaloniaSession.cs @@ -0,0 +1,108 @@ +using System.Diagnostics.CodeAnalysis; +using Avalonia.Controls; +using Avalonia.Headless; +using Avalonia.LogicalTree; +using Avalonia.Threading; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// The one headless Avalonia session of the process, shared by every benchmark class, and the helpers the +/// benchmark bodies use on its dispatcher thread. +/// +internal static class AvaloniaSession +{ + private static HeadlessUnitTestSession? session; + + /// + /// Empties , collects the garbage and runs the dispatcher's pending work, outside + /// the measurement, as an application's dispatcher is idle between two user actions; without this, the views + /// an earlier operation created would still be subscribed to Localizer.Current in the next one. + /// + /// The window to empty. + [SuppressMessage( + "Major Code Smell", + "S1215:\"GC.Collect\" should not be called", + Justification = "Weak subscriptions are only removed once their subscribers were collected." + )] + public static void EmptyWindow(Window window) => + Run(() => + { + window.Content = null; + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Dispatcher.UIThread.RunJobs(); + }); + + /// Reads the text of every below , on the UI thread. + /// The panel a benchmark returned. + /// The texts, in logical-tree order. + public static IReadOnlyList GetTextsOfAllTextBlocks(ILogical root) => + Run(() => + (IReadOnlyList) + [ + .. root.GetLogicalDescendants() + .OfType() + .Select(static textBlock => textBlock.Text ?? string.Empty), + ] + ); + + /// + /// Puts new views into a and makes it the content of + /// . Avalonia applies a binding's value as soon as it is bound, so the texts are set + /// when this returns. + /// + /// The window. + /// The number of views. + /// Creates one view. + /// The panel with the views. + public static StackPanel Open(Window window, int viewCount, Func createView) + { + var panel = new StackPanel(); + + for (var index = 0; index < viewCount; index++) + { + panel.Children.Add(createView()); + } + + window.Content = panel; + return panel; + } + + /// + /// Runs on the session's dispatcher thread and waits for it. The call flows the + /// caller's execution context, and with it , + /// so the culture is set on the calling thread: a culture set inside ends with it. + /// + /// The type of the result. + /// The work to run on the UI thread. + /// The result of . + public static T Run(Func body) => session!.Dispatch(body, CancellationToken.None).GetAwaiter().GetResult(); + + /// Runs on the session's dispatcher thread and waits for it. + /// The work to run on the UI thread. + public static void Run(Action body) => session!.Dispatch(body, CancellationToken.None).GetAwaiter().GetResult(); + + /// + /// Shows a headless window: the window every benchmark of a class makes its views the content of. Its + /// DataContext is the one the argument bindings read. + /// + /// The shown window. + public static Window ShowWindow() + { + var window = new Window + { + DataContext = ArgumentValues.Instance, + Height = 600, + Width = 800, + }; + window.Show(); + Dispatcher.UIThread.RunJobs(DispatcherPriority.Normal); + return window; + } + + /// Starts the session, once per process; each class's [GlobalSetup] calls it. + public static void Start() => session ??= HeadlessUnitTestSession.StartNew(typeof(BenchmarkApp)); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs new file mode 100644 index 0000000..9bab7a5 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/BenchmarkApp.cs @@ -0,0 +1,21 @@ +using Avalonia; +using Avalonia.Headless; +using Avalonia.Themes.Fluent; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +/// +/// The headless application every Avalonia benchmark runs in. It has the Fluent theme, as a desktop application +/// does, so that every created control gets its theme styles, as WPF's controls get their default theme styles. +/// Templates are only applied by layout, which no benchmark runs. +/// +public sealed class BenchmarkApp : Application +{ + /// The entry point HeadlessUnitTestSession.StartNew calls to configure the application. + /// The configured application builder. + public static AppBuilder BuildAvaloniaApp() => + AppBuilder.Configure().UseHeadless(new() { UseHeadlessDrawing = true }); + + /// + public override void Initialize() => this.Styles.Add(new FluentTheme()); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs new file mode 100644 index 0000000..9d77a1a --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks; + +internal static class Program +{ + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj b/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj new file mode 100644 index 0000000..73423cd --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/ResXLocalization.Avalonia.Benchmarks.csproj @@ -0,0 +1,27 @@ + + + + + true + $(CurrentTargetFramework) + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml new file mode 100644 index 0000000..be4f542 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs new file mode 100644 index 0000000..0d9db18 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/ArgumentsView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key and two <l:Argument> children bound to the DataContext. +public partial class ArgumentsView : UserControl +{ + public ArgumentsView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml new file mode 100644 index 0000000..e43a911 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs new file mode 100644 index 0000000..f9429e7 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/EnumView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:LocalizeEnum} of a different Priority member. +public partial class EnumView : UserControl +{ + public EnumView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml new file mode 100644 index 0000000..b3c7f8d --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs new file mode 100644 index 0000000..314f7d6 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/LocalizeView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key. +public partial class LocalizeView : UserControl +{ + public LocalizeView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml new file mode 100644 index 0000000..f0723d9 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs new file mode 100644 index 0000000..3ceda12 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/StaticView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {x:Static} of the designer class's property: the baseline of the forms and of opening a view. +public partial class StaticView : UserControl +{ + public StaticView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml new file mode 100644 index 0000000..4d89e91 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs new file mode 100644 index 0000000..a18f005 --- /dev/null +++ b/benchmarks/ResXLocalization.Avalonia.Benchmarks/Views/TranslationSourceView.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; + +namespace RentADeveloper.ResXLocalization.Avalonia.Benchmarks.Views; + +/// A view with 10 text elements. Every text binds the indexer of the hand-written TranslationSource: the baseline of the culture switch. +public partial class TranslationSourceView : UserControl +{ + public TranslationSourceView() => this.InitializeComponent(); + + private void InitializeComponent() => AvaloniaXamlLoader.Load(this); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs new file mode 100644 index 0000000..900b852 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/ArgumentBenchmarks.cs @@ -0,0 +1,54 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// A typed-key lookup with the two format arguments of in each +/// form, against +/// over the designer class's property. +/// +[Config(typeof(BenchmarkConfig))] +public class ArgumentBenchmarks +{ + private static readonly ArgumentValues Values = ArgumentValues.Instance; + + [Benchmark] + public string GetWithFromObject() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromObject(new { Values.Name, Values.Count }) + ); + + [Benchmark] + public string GetWithFromTuples() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromTuples((nameof(Values.Name), Values.Name), (nameof(Values.Count), Values.Count)) + ); + + [Benchmark] + public string GetWithFromValues() => + Localizer.Current.Get( + BenchmarkStringsKeys.UserMessages, + MessageArguments.FromValues(Values.Name, Values.Count) + ); + + [GlobalSetup] + public void Setup() + { + BenchmarkCulture.Use(BenchmarkCulture.German); + + var expected = string.Format( + BenchmarkCulture.German, + ExpectedViewTexts.Text(BenchmarkCulture.German, "UserMessages"), + Values.Name, + Values.Count + ); + SelfCheck.Verify(this, _ => expected); + } + + [Benchmark(Baseline = true)] + public string StringFormatDesignerProperty() => + string.Format(BenchmarkCulture.German, BenchmarkStrings.UserMessages, Values.Name, Values.Count); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs new file mode 100644 index 0000000..f103c9a --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/CodeLookupBenchmarks.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// One per lookup mode, in de-DE, against the +/// designer class's property. +/// +[Config(typeof(BenchmarkConfig))] +public class CodeLookupBenchmarks +{ + [Benchmark] + public string GetEnum() => Localizer.Current.Get(Priority.Priority1); + + [Benchmark] + public string GetStringKeyScoped() => + Localizer.Current.Get("Greeting", resourceManager: BenchmarkStringsKeys.ResourceManager); + + [Benchmark] + public string GetStringKeySearchAll() => Localizer.Current.Get("Greeting"); + + [Benchmark] + public string GetTypedKey() => Localizer.Current.Get(BenchmarkStringsKeys.Greeting); + + [Benchmark(Baseline = true)] + public string ResXDesignerProperty() => BenchmarkStrings.Greeting; + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Text( + BenchmarkCulture.German, + name == nameof(GetEnum) ? "Enum_Priority_Priority1" : "Greeting" + ) + ); + } +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs new file mode 100644 index 0000000..b2ab4b1 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/MessageFormatBenchmarks.cs @@ -0,0 +1,57 @@ +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.MessageFormat; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// One ICU message with two named arguments, through the localizer with UseMessageFormat() installed, +/// against formatting the designer class's property with the MessageFormat library directly. Both sides use a +/// formatter with the library's defaults, so pattern caching is the same, and the ratio is the cost of +/// ResXLocalization.MessageFormat itself: building the argument dictionary, the culture substitution and the +/// call through the localizer. +/// +[Config(typeof(BenchmarkConfig))] +public class MessageFormatBenchmarks +{ + private static readonly ArgumentValues Values = ArgumentValues.Instance; + + private readonly Jeffijoe.MessageFormat.MessageFormatter messageFormatter = new(); + + private readonly Dictionary values = new(StringComparer.Ordinal) + { + [nameof(Values.Count)] = Values.Count, + [nameof(Values.Name)] = Values.Name, + }; + + /// Restores the default formatter for the classes that run after this one in the same process. + [GlobalCleanup] + public void Cleanup() => Localizer.Current.MessageFormatter = StringFormatMessageFormatter.Instance; + + [Benchmark] + public string GetWithMessageFormat() => + Localizer.Current.Get( + BenchmarkStringsKeys.FilesDeleted, + MessageArguments.FromTuples((nameof(Values.Count), Values.Count), (nameof(Values.Name), Values.Name)) + ); + + [Benchmark(Baseline = true)] + public string MessageFormatterDirect() => + this.messageFormatter.FormatMessage(BenchmarkStrings.FilesDeleted, this.values, BenchmarkCulture.German); + + [GlobalSetup] + public void Setup() + { + BenchmarkCulture.Use(BenchmarkCulture.German); + Localizer.Current.UseMessageFormat(); + + // The expected text is the satellite's pattern formatted by a separate formatter, so both rows have to + // produce what the MessageFormat library makes of the German text. + var expected = new Jeffijoe.MessageFormat.MessageFormatter().FormatMessage( + ExpectedViewTexts.Text(BenchmarkCulture.German, "FilesDeleted"), + this.values, + BenchmarkCulture.German + ); + SelfCheck.Verify(this, _ => expected); + } +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs b/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs new file mode 100644 index 0000000..c8903e2 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +internal static class Program +{ + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj b/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj new file mode 100644 index 0000000..9c4dd1a --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/ResXLocalization.Core.Benchmarks.csproj @@ -0,0 +1,23 @@ + + + + $(CurrentTargetFramework) + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx b/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx new file mode 100644 index 0000000..41a9ee6 --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/Resources/Filler.resx @@ -0,0 +1,19 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Filler + + diff --git a/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs b/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs new file mode 100644 index 0000000..f51d83f --- /dev/null +++ b/benchmarks/ResXLocalization.Core.Benchmarks/SearchAllBenchmarks.cs @@ -0,0 +1,70 @@ +using System.Resources; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.Core.Benchmarks; + +/// +/// A search-all lookup whose key exists only in the last of registered resource +/// managers, against the loop an application writes without the library when it does not know which file holds +/// a key: on each of the same +/// managers, in the same order, until one has it. Both rows do the same resource-manager work, so the ratio is +/// what the library adds to a search. +/// +/// +/// Every manager before the last is a separate over Filler.resx: the +/// localizer tells managers apart by instance, and each instance has its own resource-set cache, so each miss +/// walks the whole culture fallback chain, de-DE -> de -> neutral, as a different file would. +/// +[Config(typeof(BenchmarkConfig))] +public class SearchAllBenchmarks +{ + private ResourceManager[] resourceManagers = []; + + [Params(1, 5, 20)] + public int ManagerCount { get; set; } + + [Benchmark] + public string GetStringKeySearchAll() => Localizer.Current.Get("Greeting"); + + [Benchmark(Baseline = true)] + public string HandWrittenSearchAll() + { + foreach (var resourceManager in this.resourceManagers) + { + if (resourceManager.GetString("Greeting", BenchmarkCulture.German) is { } text) + { + return text; + } + } + + return "Greeting"; + } + + [GlobalSetup] + public void Setup() + { + this.resourceManagers = + [ + .. Enumerable + .Range(1, this.ManagerCount - 1) + .Select(static _ => new ResourceManager( + FillerKeys.ResourceManager.BaseName, + typeof(FillerKeys).Assembly + )), + BenchmarkStringsKeys.ResourceManager, + ]; + + Localizer.Current.ClearResourceManagers(); + + foreach (var resourceManager in this.resourceManagers) + { + Localizer.Current.RegisterResourceManager(resourceManager); + } + + BenchmarkCulture.Use(BenchmarkCulture.German); + + var expected = ExpectedViewTexts.Text(BenchmarkCulture.German, "Greeting"); + SelfCheck.Verify(this, _ => expected); + } +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs new file mode 100644 index 0000000..673b160 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Program.cs @@ -0,0 +1,15 @@ +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +internal static class Program +{ + /// + /// [STAThread], because the in-process toolchain runs every benchmark on this thread, and WPF + /// objects need an STA thread. + /// + /// The command-line arguments, passed to BenchmarkDotNet as given. + /// 0 when every selected benchmark produced a result, otherwise 1. + [STAThread] + public static int Main(string[] args) => BenchmarkProgram.Run(typeof(Program).Assembly, args); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj b/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj new file mode 100644 index 0000000..1b27056 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/ResXLocalization.WPF.Benchmarks.csproj @@ -0,0 +1,27 @@ + + + + + $(CurrentWindowsTargetFramework) + true + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml new file mode 100644 index 0000000..08f7a58 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs new file mode 100644 index 0000000..e34e2d4 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/ArgumentsView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key and two <l:Argument> children bound to the DataContext. +public partial class ArgumentsView : UserControl +{ + public ArgumentsView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml new file mode 100644 index 0000000..a2e6575 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs new file mode 100644 index 0000000..e2da881 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/EnumView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:LocalizeEnum} of a different Priority member. +public partial class EnumView : UserControl +{ + public EnumView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml new file mode 100644 index 0000000..7a794a2 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs new file mode 100644 index 0000000..72bfc05 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/LocalizeView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {l:Localize} with a typed key. +public partial class LocalizeView : UserControl +{ + public LocalizeView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml new file mode 100644 index 0000000..25630f5 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs new file mode 100644 index 0000000..36284b8 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/StaticView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text is {x:Static} of the designer class's property: the baseline of the forms and of opening a view. +public partial class StaticView : UserControl +{ + public StaticView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml new file mode 100644 index 0000000..f13f99e --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs new file mode 100644 index 0000000..d426cb2 --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/Views/TranslationSourceView.xaml.cs @@ -0,0 +1,9 @@ +using System.Windows.Controls; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +/// A view with 10 text elements. Every text binds the indexer of the hand-written TranslationSource: the baseline of the culture switch. +public partial class TranslationSourceView : UserControl +{ + public TranslationSourceView() => this.InitializeComponent(); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs new file mode 100644 index 0000000..ee6cadd --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfFormsBenchmarks.cs @@ -0,0 +1,74 @@ +using System.Windows; +using System.Windows.Controls; +using BenchmarkDotNet.Attributes; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// Opening 100 text elements that all use one form - 10 instances of a 10-element view as a window's content - +/// against the same layout with {x:Static} of the designer class's properties. +/// +/// +/// 300 warmup iterations instead of the configuration's 3, where Avalonia needs 50: an iteration here is one +/// operation, and WPF ships as ReadyToRun code, which the tiered JIT recompiles to optimized code method by +/// method on a background thread - thousands of WPF methods. With 50 warmups the first row of the process still +/// measured about 2 ms more than the same row run last (Static: 3.4 ms against 1.4 ms); with 300 every row +/// measures the same wherever it runs. See also the call-counting delay in benchmarks/Directory.Build.props. +/// +[Config(typeof(BenchmarkConfig))] +[WarmupCount(300)] +public class WpfFormsBenchmarks +{ + private const int ViewCount = 10; + + private Window window = null!; + + [GlobalCleanup] + public void Cleanup() => this.window.Close(); + + [IterationCleanup] + public void Idle() => WpfWindow.EmptyWindow(this.window); + + [Benchmark] + [STAThread] + public StackPanel Localize() => WpfWindow.Open(this.window, ViewCount, static () => new LocalizeView()); + + [Benchmark] + [STAThread] + public StackPanel LocalizeEnum() => WpfWindow.Open(this.window, ViewCount, static () => new EnumView()); + + [Benchmark] + [STAThread] + public StackPanel LocalizeWithArguments() => + WpfWindow.Open(this.window, ViewCount, static () => new ArgumentsView()); + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + Localizer.Current.RegisterResourceManager(BenchmarkStringsKeys.ResourceManager); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = WpfWindow.Show(); + + SelfCheck.Verify( + this, + static name => + ExpectedViewTexts.Repeat( + name switch + { + nameof(LocalizeWithArguments) => ExpectedViewTexts.GermanWithArguments, + nameof(LocalizeEnum) => ExpectedViewTexts.GermanEnum, + _ => ExpectedViewTexts.German, + }, + ViewCount + ), + WpfWindow.GetTextsOfAllTextBlocks + ); + } + + [Benchmark(Baseline = true)] + [STAThread] + public StackPanel Static() => WpfWindow.Open(this.window, ViewCount, static () => new StaticView()); +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs new file mode 100644 index 0000000..3f43a2d --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfScalingBenchmarks.cs @@ -0,0 +1,118 @@ +using System.Windows; +using System.Windows.Controls; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using RentADeveloper.ResXLocalization.Benchmarks; +using RentADeveloper.ResXLocalization.WPF.Benchmarks.Views; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// Opening a view with {l:Localize} elements, against {x:Static}, and +/// switching the culture while they are live, against the hand-written . The +/// elements are / 10 instances of a 10-element view in one window. +/// +/// +/// Every iteration is one operation (InvocationCount = 1), which the culture switch needs for its +/// [IterationSetup] and which keeps the 10,000-element rows short. 5 measured iterations instead of the +/// configuration's 15, because an iteration of a 10,000-element row takes up to a second plus its setup and +/// cleanup. +/// +[Config(typeof(BenchmarkConfig))] +[CategoriesColumn] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[InvocationCount(1)] +[IterationCount(5)] +public class WpfScalingBenchmarks +{ + private const string OpenCategory = "Open"; + + private const string SwitchCategory = "Switch"; + + private Window window = null!; + + [Params(100, 10_000)] + public int ElementCount { get; set; } + + private int ViewCount => this.ElementCount / 10; + + [GlobalCleanup] + public void Cleanup() => this.window.Close(); + + [IterationCleanup] + public void Idle() => WpfWindow.EmptyWindow(this.window); + + [Benchmark] + [BenchmarkCategory(OpenCategory)] + [STAThread] + public StackPanel OpenLocalize() => WpfWindow.Open(this.window, this.ViewCount, static () => new LocalizeView()); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(OpenCategory)] + [STAThread] + public StackPanel OpenStatic() => WpfWindow.Open(this.window, this.ViewCount, static () => new StaticView()); + + [GlobalSetup] + public void Setup() + { + Localizer.Current.ClearResourceManagers(); + BenchmarkCulture.Use(BenchmarkCulture.German); + this.window = WpfWindow.Show(); + + SelfCheck.Verify( + this, + name => + ExpectedViewTexts.Repeat( + name.StartsWith(SwitchCategory, StringComparison.Ordinal) + ? ExpectedViewTexts.English + : ExpectedViewTexts.German, + this.ViewCount + ), + WpfWindow.GetTextsOfAllTextBlocks + ); + } + + [IterationSetup(Targets = [nameof(SwitchLocalize)])] + public void SetupLocalizeViews() => this.SetupViews(static () => new LocalizeView()); + + [IterationSetup(Targets = [nameof(SwitchTranslationSource)])] + public void SetupTranslationSourceViews() => this.SetupViews(static () => new TranslationSourceView()); + + [Benchmark] + [BenchmarkCategory(SwitchCategory)] + [STAThread] + public StackPanel SwitchLocalize() => + this.Switch(static () => Localizer.Current.CurrentCulture = BenchmarkCulture.English); + + [Benchmark(Baseline = true)] + [BenchmarkCategory(SwitchCategory)] + [STAThread] + public StackPanel SwitchTranslationSource() => + this.Switch(static () => TranslationSource.Instance.Culture = BenchmarkCulture.English); + + /// + /// Builds the views of a culture switch in German, outside the measurement. The previous iteration's views + /// are removed first, so that only these views are live when BenchmarkDotNet collects garbage before the + /// iteration. + /// + /// Creates one view. + private void SetupViews(Func createView) + { + this.window.Content = null; + BenchmarkCulture.Use(BenchmarkCulture.German); + _ = WpfWindow.Open(this.window, this.ViewCount, createView); + } + + /// + /// Switches the culture and processes the dispatcher's pending binding work, so every element shows the new + /// text when this returns; the self-check confirms it for all of them. + /// + /// Switches the culture. + /// The panel with the views. + private StackPanel Switch(Action switchCulture) + { + switchCulture(); + WpfWindow.Flush(); + return (StackPanel)this.window.Content; + } +} diff --git a/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs new file mode 100644 index 0000000..9895e4f --- /dev/null +++ b/benchmarks/ResXLocalization.WPF.Benchmarks/WpfWindow.cs @@ -0,0 +1,115 @@ +using System.Diagnostics.CodeAnalysis; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Threading; +using RentADeveloper.ResXLocalization.Benchmarks; + +namespace RentADeveloper.ResXLocalization.WPF.Benchmarks; + +/// +/// The window the WPF benchmarks make their views the content of, and the helpers the benchmark bodies use. +/// WPF has no headless mode, so it is a real window, shown without activation outside the visible screen +/// area. Every member runs on the benchmark thread, which is an STA thread because Main and the +/// benchmarks are [STAThread]. +/// +internal static class WpfWindow +{ + /// + /// Empties , collects the garbage and lets the dispatcher go idle, outside the + /// measurement. WPF's weak event managers remove the listeners of collected bindings only when the dispatcher + /// is idle, which an application's dispatcher is between two user actions; without this, every view an + /// earlier operation created would still be a listener of Localizer.Current in the next one. + /// + /// The window to empty. + [SuppressMessage( + "Major Code Smell", + "S1215:\"GC.Collect\" should not be called", + Justification = "The weak event managers can only remove listeners whose bindings were collected." + )] + public static void EmptyWindow(Window window) + { + window.Content = null; + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Flush(DispatcherPriority.ApplicationIdle); + } + + /// + /// Processes the dispatcher's pending binding work. The frame ends at , + /// which is above layout and rendering, so those are not part of a measurement. + /// + public static void Flush() => Flush(DispatcherPriority.DataBind); + + /// Reads the text of every below . + /// The panel a benchmark returned. + /// The texts, in logical-tree order. + public static IReadOnlyList GetTextsOfAllTextBlocks(DependencyObject root) => + [.. GetLogicalDescendants(root).OfType().Select(static textBlock => textBlock.Text)]; + + /// + /// Puts new views into a , makes it the content of + /// and processes the pending binding work, so the texts are set when this returns. + /// + /// The window. + /// The number of views. + /// Creates one view. + /// The panel with the views. + public static StackPanel Open(Window window, int viewCount, Func createView) + { + var panel = new StackPanel(); + + for (var index = 0; index < viewCount; index++) + { + _ = panel.Children.Add(createView()); + } + + window.Content = panel; + Flush(); + return panel; + } + + /// + /// Shows a window outside the visible screen area, without activating it. Its DataContext is the one + /// the argument bindings read. + /// + /// The shown window. + public static Window Show() + { + var window = new Window + { + DataContext = ArgumentValues.Instance, + Height = 600, + Left = -32_000, + ShowActivated = false, + ShowInTaskbar = false, + Top = -32_000, + Width = 800, + WindowStartupLocation = WindowStartupLocation.Manual, + WindowStyle = WindowStyle.None, + }; + window.Show(); + Flush(); + return window; + } + + private static void Flush(DispatcherPriority priority) + { + var frame = new DispatcherFrame(); + _ = Dispatcher.CurrentDispatcher.BeginInvoke(priority, new Action(() => frame.Continue = false)); + Dispatcher.PushFrame(frame); + } + + private static IEnumerable GetLogicalDescendants(DependencyObject root) + { + foreach (var child in LogicalTreeHelper.GetChildren(root).OfType()) + { + yield return child; + + foreach (var descendant in GetLogicalDescendants(child)) + { + yield return descendant; + } + } + } +} diff --git a/benchmarks/Shared/ArgumentValues.cs b/benchmarks/Shared/ArgumentValues.cs new file mode 100644 index 0000000..b8b17ae --- /dev/null +++ b/benchmarks/Shared/ArgumentValues.cs @@ -0,0 +1,14 @@ +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The two format arguments of every lookup with arguments: the values the Core argument benchmarks pass, and +/// the view model the two <l:Argument> children of every element in ArgumentsView bind to. +/// +public sealed class ArgumentValues +{ + /// Gets the one instance the benchmarks use. + public static ArgumentValues Instance { get; } = new(); + + public int Count { get; } = 3; + public string Name { get; } = "Ada"; +} diff --git a/benchmarks/Shared/BenchmarkConfig.cs b/benchmarks/Shared/BenchmarkConfig.cs new file mode 100644 index 0000000..9216e52 --- /dev/null +++ b/benchmarks/Shared/BenchmarkConfig.cs @@ -0,0 +1,77 @@ +using System.Reflection; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Perfolizer.Horology; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The one configuration every benchmark class uses, through [Config(typeof(BenchmarkConfig))], locally +/// and in CI. +/// +/// +/// +/// One short job: 3 warmup and 15 measured iterations of 100 ms, instead of BenchmarkDotNet's 500 ms and its +/// automatic choice of the iteration count. 15 iterations keep the mean time stable enough for the alert at 150% +/// in classes that run one operation per iteration, which is every UI class; the scaling classes, whose +/// 10,000-element operations are slow, lower it. Allocations, which fail the check, do not depend on the number +/// of iterations. The warmups stay, so that every measurement runs on fully optimized code, which allocates some +/// objects on the stack that unoptimized code allocates on the heap. +/// +/// +/// The in-process toolchain runs every benchmark in this process instead of a separately built and started +/// one, which removes a build and a process start per benchmark. +/// +/// +public sealed class BenchmarkConfig : ManualConfig +{ + public BenchmarkConfig() + { + this.AddJob( + Job.Default.WithToolchain(InProcessEmitToolchain.Instance) + .WithWarmupCount(3) + .WithIterationCount(15) + .WithIterationTime(TimeInterval.FromMilliseconds(100)) + ); + this.AddDiagnoser(MemoryDiagnoser.Default); + + // The full JSON report is what CI reads. The GitHub markdown and HTML reports, which CI publishes, come + // from BenchmarkDotNet's default configuration, which every [Config] is merged with; adding them here + // too would make BenchmarkDotNet warn about duplicate exporters. + this.AddExporter(JsonExporter.Full); + + // artifacts/ is where the repository puts everything it generates: .gitignore excludes it and + // scripts/clean-build-artifacts.ps1 deletes it. BenchmarkDotNet adds results/ and the logs below it. + this.ArtifactsPath = Path.Combine( + FindRepositoryRoot(), + "artifacts", + "benchmarks", + Assembly.GetEntryAssembly()!.GetName().Name! + ); + } + + /// Finds the repository root: the nearest directory above the binaries with the solution file. + /// The full path of the repository root. + /// The binaries are not below the repository root. + private static string FindRepositoryRoot() + { + for ( + var directory = new DirectoryInfo(AppContext.BaseDirectory); + directory is not null; + directory = directory.Parent + ) + { + if (File.Exists(Path.Combine(directory.FullName, "ResXLocalization.slnx"))) + { + return directory.FullName; + } + } + + throw new InvalidOperationException( + $"No directory above {AppContext.BaseDirectory} contains ResXLocalization.slnx. Run the benchmarks from their build output in the repository." + ); + } +} diff --git a/benchmarks/Shared/BenchmarkCulture.cs b/benchmarks/Shared/BenchmarkCulture.cs new file mode 100644 index 0000000..edf502c --- /dev/null +++ b/benchmarks/Shared/BenchmarkCulture.cs @@ -0,0 +1,30 @@ +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// The cultures the benchmarks look up in, and the setup that makes every side use the same one. +internal static class BenchmarkCulture +{ + /// The culture the culture-switch benchmarks switch to and from. + public static readonly CultureInfo English = CultureInfo.GetCultureInfo("en"); + + /// + /// The culture every benchmark looks up in, as most applications ship: a specific culture with a satellite + /// for its language only, so every lookup walks de-DE -> de. + /// + public static readonly CultureInfo German = CultureInfo.GetCultureInfo("de-DE"); + + /// + /// Sets the culture of every side a benchmark compares to : the designer class, + /// which reads while its Culture property is unset, + /// and . Setups call it on the thread + /// the benchmark runs on; a culture-switch benchmark instead sets only the side it measures. + /// + /// The culture every side looks up in. + public static void Use(CultureInfo culture) + { + CultureInfo.CurrentUICulture = culture; + Localizer.Current.CurrentCulture = culture; + TranslationSource.Instance.Culture = culture; + } +} diff --git a/benchmarks/Shared/BenchmarkProgram.cs b/benchmarks/Shared/BenchmarkProgram.cs new file mode 100644 index 0000000..23cb137 --- /dev/null +++ b/benchmarks/Shared/BenchmarkProgram.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using BenchmarkDotNet.Running; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// The entry point every benchmark project's Main delegates to. +internal static class BenchmarkProgram +{ + /// + /// Runs the benchmarks of that the command line selects, for example with + /// --filter *CodeLookup*, or all of them without arguments, instead of BenchmarkDotNet's interactive + /// prompt. + /// + /// The benchmark project's assembly. + /// The command-line arguments, passed to BenchmarkDotNet as given. + /// + /// 0 when every selected benchmark produced a result; 1 when one failed, such as a self-check, so that + /// CI fails instead of reporting a table with a missing row, and 1 when nothing ran, such as arguments + /// without --filter, which make BenchmarkDotNet wait for a selection that never comes. + /// + public static int Run(Assembly assembly, string[] args) + { + var summaries = BenchmarkSwitcher + .FromAssembly(assembly) + .Run(args.Length == 0 ? ["--filter", "*"] : args) + .ToList(); + + return + summaries.Count == 0 + || summaries.Exists(static summary => + summary.HasCriticalValidationErrors || summary.Reports.Any(static report => !report.Success) + ) + ? 1 + : 0; + } +} diff --git a/benchmarks/Shared/ExpectedViewTexts.cs b/benchmarks/Shared/ExpectedViewTexts.cs new file mode 100644 index 0000000..1b9b634 --- /dev/null +++ b/benchmarks/Shared/ExpectedViewTexts.cs @@ -0,0 +1,87 @@ +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The texts the self-checks expect, read from the project's BenchmarkStrings resx files: the neutral file +/// for English and the .de satellite for German. A German text is read from the satellite alone, without +/// falling back to the neutral file, so a missing satellite fails the self-check instead of letting every side +/// of a benchmark show English. +/// +internal static class ExpectedViewTexts +{ + /// The keys of every 10-element view except the enum view, element by element. + private static readonly string[] ViewKeys = + [ + "Title", + "Save", + "Cancel", + "Open", + "Close", + "Delete", + "UserMessages", + "Enum_Priority_Priority1", + "Enum_Priority_Priority2", + "Enum_Priority_Priority3", + ]; + + /// + /// Gets the English texts of a view without arguments, after a culture switch. They are the neutral file's, + /// which holds under the invariant culture: there is no + /// .en satellite, so falls back to the neutral file at run + /// time, and reading en without fallback would find nothing. + /// + public static IReadOnlyList English => Texts(CultureInfo.InvariantCulture, ViewKeys); + + /// Gets the German texts of a view without arguments: {x:Static}, {l:Localize}, TranslationSource. + public static IReadOnlyList German => Texts(BenchmarkCulture.German, ViewKeys); + + /// Gets the German texts of the enum view, one per member. + public static IReadOnlyList GermanEnum => + Texts( + BenchmarkCulture.German, + [.. Enum.GetValues().Select(static value => $"Enum_Priority_{value}")] + ); + + /// Gets the German texts of the view whose elements all have the arguments of . + public static IReadOnlyList GermanWithArguments => + [ + .. German.Select(static text => + string.Format( + BenchmarkCulture.German, + text, + ArgumentValues.Instance.Name, + ArgumentValues.Instance.Count + ) + ), + ]; + + /// The texts of instances of a view, one after the other. + /// The texts of one view. + /// The number of views. + /// The texts of all views, in order. + public static IReadOnlyList Repeat(IReadOnlyList view, int viewCount) => + [.. Enumerable.Repeat(view, viewCount).SelectMany(static texts => texts)]; + + /// + /// Reads the text of from the resx file of 's language, or + /// from the neutral file for , without falling back to another file. + /// + /// A culture, such as de-DE, or . + /// The resource key. + /// The text. + /// The file or the key does not exist. + public static string Text(CultureInfo culture, string key) + { + var fileCulture = + culture.IsNeutralCulture || culture.Equals(CultureInfo.InvariantCulture) ? culture : culture.Parent; + var resourceSet = + BenchmarkStrings.ResourceManager.GetResourceSet(fileCulture, createIfNotExists: true, tryParents: false) + ?? throw new InvalidOperationException($"BenchmarkStrings has no resource file for '{fileCulture.Name}'."); + + return resourceSet.GetString(key) + ?? throw new InvalidOperationException($"BenchmarkStrings ({fileCulture.Name}) has no key '{key}'."); + } + + private static string[] Texts(CultureInfo culture, string[] keys) => [.. keys.Select(key => Text(culture, key))]; +} diff --git a/benchmarks/Shared/Priority.cs b/benchmarks/Shared/Priority.cs new file mode 100644 index 0000000..05d8ec0 --- /dev/null +++ b/benchmarks/Shared/Priority.cs @@ -0,0 +1,20 @@ +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The enumeration the enum lookups localize, through the keys Enum_Priority_Priority1 to +/// Enum_Priority_Priority10 of BenchmarkStrings.resx. It has 10 members so that the 10 elements +/// of the enum view each show a different text, as the other views do. +/// +public enum Priority +{ + Priority1 = 1, + Priority2 = 2, + Priority3 = 3, + Priority4 = 4, + Priority5 = 5, + Priority6 = 6, + Priority7 = 7, + Priority8 = 8, + Priority9 = 9, + Priority10 = 10, +} diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs b/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs new file mode 100644 index 0000000..4043ea2 --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.Designer.cs @@ -0,0 +1,234 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace RentADeveloper.ResXLocalization.Benchmarks { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "18.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class BenchmarkStrings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal BenchmarkStrings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings", typeof(BenchmarkStrings).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Cancel. + /// + public static string Cancel { + get { + return ResourceManager.GetString("Cancel", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Close. + /// + public static string Close { + get { + return ResourceManager.GetString("Close", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Delete. + /// + public static string Delete { + get { + return ResourceManager.GetString("Delete", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 1. + /// + public static string Enum_Priority_Priority1 { + get { + return ResourceManager.GetString("Enum_Priority_Priority1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 10. + /// + public static string Enum_Priority_Priority10 { + get { + return ResourceManager.GetString("Enum_Priority_Priority10", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 2. + /// + public static string Enum_Priority_Priority2 { + get { + return ResourceManager.GetString("Enum_Priority_Priority2", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 3. + /// + public static string Enum_Priority_Priority3 { + get { + return ResourceManager.GetString("Enum_Priority_Priority3", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 4. + /// + public static string Enum_Priority_Priority4 { + get { + return ResourceManager.GetString("Enum_Priority_Priority4", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 5. + /// + public static string Enum_Priority_Priority5 { + get { + return ResourceManager.GetString("Enum_Priority_Priority5", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 6. + /// + public static string Enum_Priority_Priority6 { + get { + return ResourceManager.GetString("Enum_Priority_Priority6", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 7. + /// + public static string Enum_Priority_Priority7 { + get { + return ResourceManager.GetString("Enum_Priority_Priority7", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 8. + /// + public static string Enum_Priority_Priority8 { + get { + return ResourceManager.GetString("Enum_Priority_Priority8", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Priority 9. + /// + public static string Enum_Priority_Priority9 { + get { + return ResourceManager.GetString("Enum_Priority_Priority9", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {Count, plural, one {# file deleted} other {# files deleted}} by {Name}. + /// + public static string FilesDeleted { + get { + return ResourceManager.GetString("FilesDeleted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Hello. + /// + public static string Greeting { + get { + return ResourceManager.GetString("Greeting", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Open. + /// + public static string Open { + get { + return ResourceManager.GetString("Open", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Save. + /// + public static string Save { + get { + return ResourceManager.GetString("Save", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Settings. + /// + public static string Title { + get { + return ResourceManager.GetString("Title", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} has {1} new messages. + /// + public static string UserMessages { + get { + return ResourceManager.GetString("UserMessages", resourceCulture); + } + } + } +} diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.de.resx b/benchmarks/Shared/Resources/BenchmarkStrings.de.resx new file mode 100644 index 0000000..8a0e79c --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.de.resx @@ -0,0 +1,73 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Hallo + + + Einstellungen + + + Speichern + + + Abbrechen + + + Öffnen + + + Schließen + + + Löschen + + + {0} hat {1} neue Nachrichten + + + {Count, plural, one {# Datei} other {# Dateien}} von {Name} gelöscht + + + Priorität 1 + + + Priorität 2 + + + Priorität 3 + + + Priorität 4 + + + Priorität 5 + + + Priorität 6 + + + Priorität 7 + + + Priorität 8 + + + Priorität 9 + + + Priorität 10 + + diff --git a/benchmarks/Shared/Resources/BenchmarkStrings.resx b/benchmarks/Shared/Resources/BenchmarkStrings.resx new file mode 100644 index 0000000..1a48b76 --- /dev/null +++ b/benchmarks/Shared/Resources/BenchmarkStrings.resx @@ -0,0 +1,73 @@ + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Hello + + + Settings + + + Save + + + Cancel + + + Open + + + Close + + + Delete + + + {0} has {1} new messages + + + {Count, plural, one {# file deleted} other {# files deleted}} by {Name} + + + Priority 1 + + + Priority 2 + + + Priority 3 + + + Priority 4 + + + Priority 5 + + + Priority 6 + + + Priority 7 + + + Priority 8 + + + Priority 9 + + + Priority 10 + + diff --git a/benchmarks/Shared/SelfCheck.cs b/benchmarks/Shared/SelfCheck.cs new file mode 100644 index 0000000..597e196 --- /dev/null +++ b/benchmarks/Shared/SelfCheck.cs @@ -0,0 +1,76 @@ +using System.Reflection; +using BenchmarkDotNet.Attributes; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// Runs every benchmark method of a class once and throws when it does not produce the expected texts. Each +/// class calls it from its [GlobalSetup], so a broken lookup stops that class with an error naming the +/// benchmark, instead of producing a number for the failure text. +/// +internal static class SelfCheck +{ + /// Verifies the benchmark methods of that return the text itself. + /// The benchmark class instance. + /// The text each benchmark method, by name, has to return. + public static void Verify(object benchmarks, Func expected) => + Verify(benchmarks, name => [expected(name)], static result => [result]); + + /// + /// Verifies every benchmark method of , after running the + /// [IterationSetup] methods that apply to it. + /// + /// The type every benchmark method of the class returns. + /// The benchmark class instance. + /// The texts each benchmark method, by name, has to produce, in order. + /// Reads the texts out of what a benchmark method returned. + /// A benchmark method produced other texts. + public static void Verify( + object benchmarks, + Func> expected, + Func> texts + ) + { + var type = benchmarks.GetType(); + var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance); + var iterationSetups = methods + .Select(static method => + (Method: method, SetupAttribute: method.GetCustomAttribute()) + ) + .Where(static setup => setup.SetupAttribute is not null) + .ToList(); + + foreach (var method in methods.Where(static method => method.IsDefined(typeof(BenchmarkAttribute)))) + { + foreach (var setup in iterationSetups) + { + if (setup.SetupAttribute?.Targets is not { Length: > 0 } targets || targets.Contains(method.Name)) + { + _ = setup.Method.Invoke(benchmarks, BindingFlags.DoNotWrapExceptions, null, null, null); + } + } + + var actual = texts((TResult)method.Invoke(benchmarks, BindingFlags.DoNotWrapExceptions, null, null, null)!); + var wanted = expected(method.Name); + + if (actual.Count != wanted.Count) + { + throw new InvalidOperationException( + $"Self-check of {method.DeclaringType!.Name}.{method.Name} failed: expected {wanted.Count} texts, " + + $"but it produced {actual.Count}." + ); + } + + for (var index = 0; index < wanted.Count; index++) + { + if (!string.Equals(actual[index], wanted[index], StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Self-check of {method.DeclaringType!.Name}.{method.Name} failed: text {index + 1} of " + + $"{wanted.Count} is \"{actual[index]}\" instead of \"{wanted[index]}\"." + ); + } + } + } + } +} diff --git a/benchmarks/Shared/TranslationSource.cs b/benchmarks/Shared/TranslationSource.cs new file mode 100644 index 0000000..8d9c393 --- /dev/null +++ b/benchmarks/Shared/TranslationSource.cs @@ -0,0 +1,47 @@ +using System.ComponentModel; +using System.Globalization; + +namespace RentADeveloper.ResXLocalization.Benchmarks; + +/// +/// The usual hand-written way to switch languages live without ResXLocalization, and the baseline of the +/// culture switch: elements bind {Binding [Key], Source={x:Static shared:TranslationSource.Instance}}, +/// and setting makes every such binding read its text again. +/// +public sealed class TranslationSource : INotifyPropertyChanged +{ + private CultureInfo culture = CultureInfo.CurrentUICulture; + + private TranslationSource() { } + + public event PropertyChangedEventHandler? PropertyChanged; + + /// Gets the one instance the views bind to. + public static TranslationSource Instance { get; } = new(); + + /// Gets or sets the culture the indexer looks up in, and notifies every indexer binding. + public CultureInfo Culture + { + get => this.culture; + set + { + this.culture = value; + + // The name that tells every indexer binding to read again: WPF's Binding.IndexerName, "Item[]", in the + // WPF project, the only one with a -windows target framework; Avalonia's "Item" otherwise, because its + // compiled indexer bindings ignore "Item[]". One framework-specific name, not both, and not an empty + // name ("every property changed"), which both frameworks accept but handle at a higher cost: the + // baseline does no more work than an application's own source. +#if WINDOWS + this.PropertyChanged?.Invoke(this, new("Item[]")); +#else + this.PropertyChanged?.Invoke(this, new("Item")); +#endif + } + } + + /// Gets the text of in . + /// The resource key. + /// The text, or when there is none. + public string this[string key] => BenchmarkStrings.ResourceManager.GetString(key, this.culture) ?? key; +} diff --git a/docs/guides/benchmarks.md b/docs/guides/benchmarks.md new file mode 100644 index 0000000..2509d7d --- /dev/null +++ b/docs/guides/benchmarks.md @@ -0,0 +1,26 @@ +# Benchmarks + +Every push to `main` measures what ResXLocalization costs an application, on the same GitHub-hosted Windows +runner, .NET 10 and the just-in-time compiler: + +- [Latest results](https://rent-a-developer.github.io/ResXLocalization/benchmarks/latest/): the tables of the + last commit on `main`, with the hardware and runtime they were measured on. +- [History](https://rent-a-developer.github.io/ResXLocalization/benchmarks/history/): the time and allocations + of every benchmark over the last 200 commits on `main`. + +Every benchmark is compared with the code an application writes without the library: the `.resx` designer +class and hand-written code over it. + +| Benchmark | What it measures | Baseline | +| --- | --- | --- | +| `CodeLookupBenchmarks` | `Localizer.Current.Get` with a typed key, a scoped key, a search-all key and an enum value | The designer class's property, `BenchmarkStrings.Greeting` | +| `SearchAllBenchmarks` | A search-all lookup whose key is in the last of 1, 5 or 20 registered resource files | A hand-written loop that asks the same resource managers, in the same order, until one has the key | +| `ArgumentBenchmarks` | `Get` with `MessageArguments.FromValues`, `FromTuples` and `FromObject` | `string.Format` over the designer class's property | +| `MessageFormatBenchmarks` | `Get` with two named arguments and `UseMessageFormat()` installed | The MessageFormat library's `FormatMessage` over the designer class's property | +| `AvaloniaFormsBenchmarks`, `WpfFormsBenchmarks` | Opening 100 elements that all use `{l:Localize}`, `{l:Localize}` with two `` children, or `{l:LocalizeEnum}` | The same views with `{x:Static}` of the designer class's properties | +| `AvaloniaScalingBenchmarks`, `WpfScalingBenchmarks`, Open | Opening a window with 100 or 10,000 `{l:Localize}` elements | The same views with `{x:Static}` | +| `AvaloniaScalingBenchmarks`, `WpfScalingBenchmarks`, Switch | Switching the culture while 100 or 10,000 `{l:Localize}` elements are live | The same views bound to a hand-written `TranslationSource` with a string indexer | + +The **Ratio** column is a benchmark's time divided by its baseline's, so 1.20 means 20% slower than the code +without the library; **Allocated** is the memory one operation allocates, and **Alloc Ratio** compares it with +the baseline the same way. diff --git a/docs/toc.yml b/docs/toc.yml index 6f082b6..959ab21 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -18,6 +18,8 @@ href: guides/native-aot.md - name: Migrating to 2.0 href: guides/migration-2.0.md + - name: Benchmarks + href: guides/benchmarks.md - name: Reference items: - name: Generated keys diff --git a/openspec/changes/add-benchmarks/.openspec.yaml b/openspec/changes/add-benchmarks/.openspec.yaml new file mode 100644 index 0000000..1f5d280 --- /dev/null +++ b/openspec/changes/add-benchmarks/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-24 diff --git a/openspec/changes/add-benchmarks/design.md b/openspec/changes/add-benchmarks/design.md new file mode 100644 index 0000000..4731c74 --- /dev/null +++ b/openspec/changes/add-benchmarks/design.md @@ -0,0 +1,490 @@ +# Design + +## Context + +See proposal.md for the motivation and specs/performance-benchmarks/spec.md for the requirements. + +The code paths being measured: + +``` + Code: Localizer.Get(key, args, rm?) + +-- rm given? --> rm.GetString(key, culture) + | no --> loop over registered managers, first hit wins + +-- MessageFormatter.Format(value, args, culture) (runs even without arguments) + default: string.Format | ICU: Dictionary + Jeffijoe FormatMessage + + XAML: ProvideValue --> MultiBinding per element + [0] RefreshTriggeringRevision of Localizer.Current [1] key [2..] Argument values + LocalizedMessageConverter --> MessageArguments.FromTuples --> Localizer.Get + Culture switch --> PropertyChanged --> every live binding calls Get again +``` + +Constraints from the repository: + +- Projects under `tests/` inherit xUnit, coverlet and the test SDK from `tests/Directory.Build.props`, which a + BenchmarkDotNet console app must not carry. Every project still has to pass the repository's analyzers and + style gate (`scripts/tidy-code.ps1`), which the root `Directory.Build.props` imports. +- WPF builds only on Windows; `ResXLocalization.NonWindows.slnf` is what Linux builds use. +- Samples reference the source generator as an analyzer `ProjectReference`; the resx wiring comes from the + root `Directory.Build.targets`. +- GitHub Pages already deploys the DocFX site through `actions/deploy-pages` on pushes to `main`, built with + `--warningsAsErrors`, so a classic `gh-pages` branch would not be served. +- Every third-party action is pinned by commit SHA; Dependabot keeps them current. + +## Goals / Non-Goals + +**Goals:** +- Numbers that answer "will this make my app slow" and "how does it scale" with as little benchmark code as + possible. +- A regression check that gives the same result every time it runs on the same code, even though the speed + of GitHub's shared CI machines varies from run to run. +- Only existing tools: BenchmarkDotNet for measuring, `benchmark-action/github-action-benchmark` for history, + comparison, alerts and charts. + +**Non-Goals:** +- Native AOT benchmarks. The code these benchmarks measure does the same work under AOT; the existing AOT gate covers correctness. + `MessageArguments.FromObject` uses reflection and would be the one to add if AOT numbers are ever wanted. +- .NET 8 benchmarks, multiple operating systems, rendering or layout cost, startup and first-lookup cost + (satellite assembly loading), the source generator's build time. +- Comparisons with other localization libraries; they can be added later. +- Optimizing anything the benchmarks reveal; that is follow-up work. +- Running `main` and the pull request one after the other on the same machine to compare them. + +## Decisions + +### Three projects under a new `benchmarks/` folder, one per suite + +``` + benchmarks/ + Directory.Build.props imports root props, NoWarn, compiles Shared/*.cs, links + Shared/Resources/BenchmarkStrings.* into every project + Shared/ [Config], self-check, expected texts, culture setup, Priority, + ArgumentValues, TranslationSource (compiled into all three) + Shared/Resources/ BenchmarkStrings.resx, .de.resx, .Designer.cs (linked into all three) + ResXLocalization.Core.Benchmarks Core, MessageFormat $(CurrentTargetFramework) + ResXLocalization.Avalonia.Benchmarks Avalonia headless $(CurrentTargetFramework) + ResXLocalization.WPF.Benchmarks WPF $(CurrentWindowsTargetFramework) +``` + +The projects mirror the packages and the test suites. The Core suite answers "did the engine get slower" and +"what does one lookup cost"; the UI suites answer "what does a localized element cost and how does it scale". +Core needs its own suite because a lookup is a small share of a UI benchmark's time and allocations: a +lookup that doubles in cost moves a UI row by a few percent, below both CI thresholds. + +- MessageFormat stays in the Core suite: it is one class, and it measures Core lookups with the ICU + formatter installed. +- `benchmarks/Directory.Build.props` imports the root props, like `tests/` does, adds the `NoWarn` for + `CS1574;CS1591` that samples and tests carry for generated keys, and compiles `benchmarks/Shared/*.cs` into + every benchmark project, so the configuration, the self-check and the types every suite uses (`Priority`, + `ArgumentValues`, `TranslationSource`) exist once. It also links the one `BenchmarkStrings` resource file + set (see "Resource data") into every project. +- All three join `ResXLocalization.slnx` under `/04_benchmarks/`; Core and Avalonia also join the + non-Windows filter, WPF does not, as in `src/`. They are built by the existing build jobs, so a broken + benchmark fails the build, but they are not test projects and `dotnet test` ignores them. +- The UI suites have one row per extension form (typed key, `Argument`, enum), not every key type and + argument form: those are the Core suite's. + +Alternatives: Core with Avalonia and WPF separate (splits by "builds on Linux", which says nothing about what +a project measures); no Core suite (Core regressions disappear in the UI numbers, and the code-only features - +`MessageArguments`, search-all - are not reached through XAML); a fourth +project for MessageFormat (a project for one class); placing them under `tests/` (inherits the test stack). + +### Resource data + +- **One resource file set for all three suites**: `benchmarks/Shared/Resources/BenchmarkStrings.resx` with + plain texts, a text with two arguments, one ICU message with two named arguments and the 10 enum keys, a + `.de` file that translates them, and no `de-DE` file. `benchmarks/Directory.Build.props` links them into + every project, which embeds the neutral file and builds its own `de` satellite from the `.de` file; a suite + carries keys only another suite uses, which changes no number (see below). Its `LogicalName` is the + designer class's base name, `RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings`, because a linked + file would otherwise be named after the project's `RootNamespace` and its link path. The designer class + and the generated `BenchmarkStringsKeys` are in the namespace of the code under `Shared/`, which every + benchmark file already imports, so no project needs a using of its own for them. +- **Core**: Every benchmark looks up in `de-DE`, which is how most apps ship: a + specific culture with a satellite for its language. The fallback walk `de-DE -> de` is `ResourceManager`'s + and costs the baseline and the library the same; it matters for search-all, where every manager without + the key is walked through the whole chain (`de-DE -> de -> neutral`) before the next one is tried. That + walk is the cost of any search over several files, so the search-all baseline is a hand-written loop over + the same managers that pays it too (see "Baselines"). Search-all benchmarks register 0, 4 or 19 + managers without the key before `BenchmarkStrings`, for 1, 5 or 20 managers in all. They come from one + `Filler.resx` with a single other key, in the Core project only, whose `CustomToolNamespace` puts + `FillerKeys` into the benchmark classes' namespace: the benchmark creates 19 `ResourceManager` instances + over its base name. The library tells managers apart by instance, and each instance has its own resource-set cache, so + every miss is a real lookup with the full fallback walk, the same work as 19 different files. +- **Avalonia and WPF**: each 10-element view uses 10 different keys once, + so a view has different texts, as a real one does: seven texts and the first three enum keys, and the + `{l:LocalizeEnum}` view one member each of the shared `Priority` enum, whose 10 members `Priority1` to + `Priority10` exist only for that. + +The key count does not change the numbers: `ResourceManager` keeps a loaded resource set in a dictionary, so +one lookup costs the same with 10 keys or 1,000. Loading the set is a one-time startup cost and is not +measured. + +`BenchmarkStrings.resx` has a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` +emits: a public static property per key that calls `ResourceManager.GetString(name, resourceCulture)`. It is +the baseline (see "Baselines"). Typed keys come from the source generator, which handles a resx with a +designer, as in the samples. + +All resx, designer and view files are small enough to write by hand and are committed, like the samples' +files. The samples hand-write their designer files because `PublicResXFileCodeGenerator` runs only inside +Visual Studio. The data is fixed, so nothing needs to regenerate it. + +Alternatives: a script that generates large data sets (1,000 keys, 100-element views), which would run once +and then go out of date, and whose size does not change the results; a build-time generator, which would be +new tooling for a fixed data set; a resource file set per project, as each sample has its own (three copies +of the same texts, two of them identical, and the shared code reached each project's copy only through a +global using of that project's `Resources` namespace). + +### BenchmarkDotNet configuration + +- One `[Config]` shared by all classes: `MemoryDiagnoser`, the JSON (full) and GitHub-markdown and HTML + exporters, and one short job used locally and in CI. The config adds only the JSON exporter: BenchmarkDotNet + merges every `[Config]` with its default configuration, which already has the other two, and adding them + again makes it warn about duplicate exporters. + - **3 warmup and 15 measured iterations of 100 ms each**, instead of BenchmarkDotNet's 500 ms and its + automatic choice of the iteration count. Every UI class runs one operation per iteration (see "The + dispatcher goes idle between iterations"), so its iterations are its samples: with 3 of them the 99.9% + error was often larger than the mean, and two classes measuring the same 100-element view differed by + 40%. 15 iterations cost seconds per class. The scaling classes have 5 (`[IterationCount(5)]`), because an + iteration of a 10,000-element row takes up to a second plus its setup and cleanup. Time only raises an + alert at 150%; allocations, which fail the check, do not change with the number of iterations. + - **The warmups stay**: .NET 10 allocates some objects on the stack, but only in fully optimized code. A + benchmark measured before the JIT has optimized it would count allocations a running app does not have + (spec: "Measurements run on optimized code"). + - **The forms classes have 50 (Avalonia) and 300 (WPF) warmup iterations**, and the host runs **without + the tiered JIT's call-counting delay** (`System.Runtime.TieredCompilation.CallCountingDelayMs = 0` as a + `RuntimeHostConfigurationOption` in `benchmarks/Directory.Build.props`). One operation there runs + hundreds of UI-framework methods, and a method is promoted to optimized code only after 30 calls and + once no new method was compiled for 100 ms. An iteration is one operation, and every operation compiles + new methods, so the delay kept restarting: with Avalonia's Fluent theme, the first row of the first class + measured three times as slow as the same operation later in the process, with 15 warmups and with 50. + Without the delay and with 50 warmups (more than the 30 calls) every Avalonia row matches the same + operation in the scaling class. WPF ships as ReadyToRun code, which the JIT recompiles to optimized code + on a background thread, thousands of methods: with 50 warmups the first WPF row of the process still + measured about 2 ms more than the same row run last (`Static` 3.4 ms against 1.4 ms; turning the JIT's + tiering and ReadyToRun off brought it to 1.7 ms, a larger gen0 budget or non-concurrent GC changed + nothing). With 300 every WPF row measures the same wherever it runs, for about 10 s more. Dynamic PGO still runs, so code and allocations are a long-running application's; + turning tiered compilation off instead changed the Core suite's allocations (no dynamic PGO, so less + stack allocation). It is the host's configuration because the benchmarks run in-process. The scaling + classes keep 3 warmups: one operation is large enough, and they run after the forms classes. + - **In-process toolchain** (`InProcessEmitToolchain`): every benchmark runs in the BenchmarkDotNet host + process instead of a separately built and started process. That removes a build and a process start per + benchmark; the cost is that benchmarks share one process and its GC state, which does not change these + numbers noticeably. The host's `Main` is `[STAThread]` in the WPF project, so the WPF benchmarks run on + an STA thread. +- **Results go to `artifacts/benchmarks//`**: the shared config sets BenchmarkDotNet's + `ArtifactsPath` there, resolved from the repository root, instead of its default `BenchmarkDotNet.Artifacts` + folder in the working directory. `artifacts/` is where the repository puts everything it generates on + purpose: `.gitignore` already excludes it and `scripts/clean-build-artifacts.ps1` already deletes it, so + neither needs a change. +- Each class marks its library-free variant `[Baseline = true]` (see "Baselines"), so the Ratio column is the + overhead. +- `[Params]` for manager count (1, 5, 20) and element count (100, 10,000). Two element counts are enough to + show whether the cost per element stays the same as N grows. +- Each class's `[GlobalSetup]` runs every benchmark method of the class once and throws when the text is not + the expected one, which makes BenchmarkDotNet stop the class with an error (spec: "Benchmarks verify their + own results"). The expected texts are read from the resx files themselves (`ExpectedViewTexts`): German + from the `.de` satellite alone, without fallback, so a missing satellite fails the check instead of letting + every side show English. +- `Program.Main` is `BenchmarkSwitcher.FromAssembly(...).Run(args)`, so local runs can filter with + `--filter`. Without arguments it runs every benchmark (`--filter *`) instead of BenchmarkDotNet's + interactive prompt, which would wait for input. It returns 1 when a benchmark has no result, so CI fails, + and when nothing ran: arguments without `--filter` reach that prompt, which gets no input. +- **One report per class**, not `--join`: a joined table has one baseline for every class that shares a job, + so in the Core suite every Ratio was against `CodeLookupBenchmarks`' baseline. +- Method names say what a row calls, so a row reads on its own in the history chart: `ResXDesignerProperty`, + `GetTypedKey`, `GetStringKeyScoped`, `GetStringKeySearchAll`, `GetEnum`, `HandWrittenSearchAll`, + `StringFormatDesignerProperty`, + `GetWithFromValues` / `FromTuples` / `FromObject`, `MessageFormatterDirect`, `GetWithMessageFormat`; the UI + classes `Static`, `Localize`, `LocalizeWithArguments`, `LocalizeEnum`, `OpenStatic`, `OpenLocalize`, + `SwitchTranslationSource`, `SwitchLocalize`. +- Every result table has its baseline row, the search-all table included: one hand-written search row per + manager count next to the search-all row (spec: "Every result has a baseline"). + +### Baselines + +Every baseline is the code an application writes without the library; no other localization library is +benchmarked (a comparison can be added later as extra rows, outside the allocation gate). + +| Benchmark | `[Baseline = true]` | +| --- | --- | +| Code lookup (typed, scoped, search-all, enum) | `BenchmarkStrings.Greeting` (designer property) | +| Search-all with 1, 5 or 20 managers | `HandWrittenSearchAll`: `GetString(key, culture)` on the same managers, in registration order, until one has the key | +| Code lookup with arguments | `string.Format(culture, BenchmarkStrings.UserMessages, name, count)` | +| ICU MessageFormat | `messageFormatter.FormatMessage(BenchmarkStrings.FilesDeleted, values, culture)` | +| Extension forms (100 elements) | the same 10-element views with `{x:Static}` | +| Opening a view with N elements | the same views with `{x:Static}` | +| Culture switch with N elements | the same views with `{Binding [Key], Source={x:Static local:TranslationSource.Instance}}` | + +- **Search-all**: the designer property knows which file holds a key, so against it the search-all row + measured the search itself: at 20 managers 65 times the time, nearly all of it `ResourceManager`'s misses, + which any search over 20 files pays. The baseline is the loop an application writes without the library + when it does not know the file: `GetString(key, culture)` on each of the same manager instances, in the same + order the localizer has them, until one returns a value. The ratio is what the library adds to that search. + `CodeLookupBenchmarks` keeps the designer property: with one registered manager its search-all row does the + same work as the property. +- **Culture**: the designer class reads `CultureInfo.CurrentUICulture` while its `Culture` property is unset, + so every setup calls `BenchmarkCulture.Use`, which sets `CurrentUICulture`, `Localizer.Current.CurrentCulture` + and `TranslationSource.Instance.Culture` to the same culture. Only the culture-switch benchmarks set one + side directly: the side they measure, so that the other side's notification is not part of the operation. +- **ICU**: the baseline uses one `new MessageFormatter()` with the library's defaults, which is what + `UseMessageFormat()` creates, so pattern caching is the same on both sides. The ratio is the cost of + `ResXLocalization.MessageFormat` itself: building the argument dictionary, the culture substitution and the + call through the localizer. What ICU costs compared with `string.Format` is the MessageFormat library's + property and is not measured. One message is enough: plural, select and the other ICU features run inside + the MessageFormat library and cost the baseline the same. The adapter's overhead depends only on the + number of arguments and whether they are named, so the message has two named arguments, which run both + of the adapter's loops (positions and names). +- **`TranslationSource`**: the usual hand-written way to switch languages live, about 20 lines, in + `benchmarks/Shared/` for both UI suites. A singleton implementing `INotifyPropertyChanged` with + `this[string key] => BenchmarkStrings.ResourceManager.GetString(key, this.Culture)` and a `Culture` setter + that raises one `PropertyChanged` for the indexer, with the framework's name, chosen by `#if WINDOWS` (only + the WPF project has a `-windows` target framework): `"Item[]"` (`Binding.IndexerName`) in WPF and `"Item"` + in Avalonia, whose compiled indexer bindings ignore `"Item[]"` (found by the feasibility check). Raising + both would make the baseline handle two events; an empty name ("every property changed") works in both + but made the baseline's 10,000-element switch allocate three times as much in WPF. Elements bind `[key]` on it with the framework's normal binding. Its culture-switch row sets + `TranslationSource.Instance.Culture`; the library's row sets `Localizer.Current.CurrentCulture`. + +Alternatives: raw `ResourceManager.GetString` (the same cost as the designer property, but code consumers do +not recognize); Microsoft.Extensions.Localization in the Core suite (rarely used in XAML applications, so it +answers no question of this suite); WPFLocalizeExtension in the WPF suite (left for later: new releases of +a third-party project to keep up with, and no equivalent for Avalonia). + +### UI benchmarks use compiled XAML views + +Every UI row, baselines included, is a view written in XAML and compiled by the build, as in a consumer +app: + +``` + Consumer app Benchmark + +-----------------------------+ +---------------------------------------+ + | MainView.axaml / .xaml | | LocalizedView.axaml / .xaml | + | | == | "{l:Localize ...}"/> | + | compiled by the build | | compiled by the build (same tooling) | + | new MainView() -> window | | new LocalizedView() -> window | + +-----------------------------+ +---------------------------------------+ +``` + +- **Avalonia** compiles `.axaml` into IL, so `new LocalizedView()` runs the generated code that creates the + extension, calls `ProvideValue` with the real XAML context and applies the binding. +- **WPF** compiles `.xaml` to BAML, and `InitializeComponent` reads it at run time with WPF's XAML reader and + its service provider. Reading BAML is a cost consumers pay too; the baseline views pay it as well, so the + ratio still shows what `{l:Localize}` adds. +- **Views per UI project**: five 10-element views under `Views/`, written by hand, each using one form for + all its elements: + + ``` + {x:Static} baseline of both benchmarks + {l:Localize} forms and scaling + {l:Localize} + 2 forms only (arguments bound to the view's DataContext) + {l:LocalizeEnum} forms only + TranslationSource scaling only (baseline of the culture switch) + ``` + +- **Forms**: one row per view, each opening 10 instances (100 elements) of it in a window. 100 elements + spread the fixed cost of a view and a window, so a row's ratio to `{x:Static}` is what that form costs per + element in a realistic view. A benchmark with a single element is not used: no app opens one, and its + number would mostly be that fixed cost. +- **Scaling**: plain `{l:Localize}` only. The benchmark puts N / 10 instances of a 10-element view into a + `StackPanel` that is a window's content: 10 or 1,000 views for N = 100 or 10,000, like a + screen built from user controls. One form is enough because what can grow faster than the element count + is shared by all forms: every element subscribes to `Localizer.Current` through its refresh-trigger + binding, and a culture switch updates every subscribed binding. What `` and + `{l:LocalizeEnum}` add - more bindings, building the enum key - is per element and independent of the + other elements, so it is N times the constant the forms benchmark measures. A view with mixed forms costs + about the sum of its elements' per-form costs. +- **What is measured**: creating the view, making it the window's content, and ending when its texts are + set. Rendering is not measured: it is the same for every row and is the UI framework's cost, not the + library's. +- **Culture switch**: `[IterationSetup]` creates the N elements in a window; the benchmark switches the + culture (`Localizer.Current.CurrentCulture`, or `TranslationSource.Instance.Culture` for its baseline), + processes the dispatcher's pending work and checks the last element's text. +- **Avalonia** runs on the headless platform: `[GlobalSetup]` starts a `HeadlessUnitTestSession` once, every + benchmark body runs through its dispatcher, and the window is a headless window. The application has the + Fluent theme, as a desktop application does, so every control gets its theme styles, as WPF's controls get + their default theme styles. Templates are applied only by layout, which no row runs in either framework; + the library does not depend on them or on the visual tree (it has no attach or detach hooks, and + `DataContext` is inherited through the logical tree), which the self-checks confirm. The session's + dispatch flows the caller's execution context, and with it `CultureInfo.CurrentUICulture`, so the culture + is set on the BenchmarkDotNet thread: a culture set inside a dispatched body is gone in the next one. Headless replaces only + the operating system window, input, compositor and message loop; compiled XAML, controls, the property + system, bindings and the library are the same code as in a desktop app, and binding updates after + `PropertyChanged` run on the UI thread without the dispatcher. The numbers therefore show the library's + cost, but not a full frame of a real app. +- **WPF** has no headless mode and uses a real `Window`, shown with `ShowActivated = false` outside the + visible screen area. The benchmarks are marked `[STAThread]` so BenchmarkDotNet runs them on an STA + thread, and pending dispatcher work is processed with a `DispatcherFrame`. +- **Pending work is processed down to binding priority only**: WPF's frame ends at + `DispatcherPriority.DataBind`, Avalonia's `RunJobs` at `DispatcherPriority.Normal` (Avalonia's `DataBind` is + obsolete). Both are above layout and rendering, which therefore stay out of the measurement in both + frameworks. +- **The dispatcher goes idle between iterations**: every UI class has an `[IterationCleanup]` that empties the + window, collects garbage and runs the dispatcher's idle work, outside the measurement, as an application's + dispatcher is idle between two user actions. WPF removes the weak-event listeners of collected bindings only + at idle priority; without this step every view an earlier operation created was still a listener of + `Localizer.Current`, and a culture switch with 100 elements took about six times as long. Avalonia does the + same for parity. Because of the `[IterationCleanup]`, BenchmarkDotNet runs one operation per iteration in the + forms classes too, like the scaling classes. + +Alternatives: a scaling run per form (mostly parallel lines, three times the slowest part of the suite, +and the arguments form would need a library-free baseline for live switching with arguments); building the +elements in code and calling `ProvideValue` directly (not what a consumer app +runs, and WPF would need a hand-written service provider); parsing XAML at run time +(`AvaloniaRuntimeXamlLoader`, `XamlReader.Parse`), which no consumer app does; a real Avalonia desktop +window (its timing includes the operating system and rendering, which vary between runs and are not the +library's cost). + +### Regression detection with github-action-benchmark, two series + +The action reads one BenchmarkDotNet JSON file (`tool: benchmarkdotnet`) and compares only the mean time. +Each class writes its own report, so one script merges them and derives the allocation series; the job then +feeds the action twice: + +``` + Core / Avalonia / WPF *-report-full.json (one per class) + | + v + scripts/convert-benchmark-results.ps1 + +--> times.json (every report's Benchmarks array, the first report's host info) + | --> github-action-benchmark tool: benchmarkdotnet "Time" + | alert-threshold 150%, fail-on-alert false, summary-always true + | + +--> allocations.json (name + AllocatedBytes per benchmark, customSmallerIsBetter format) + --> github-action-benchmark tool: customSmallerIsBetter "Allocations" + alert-threshold 125%, summary-always true + fail-on-alert: true, false while the PR has label accept-allocation-increase +``` + +Benchmark names include their class and its namespace, which differs per project, so names from different +reports cannot collide. + +- **Pull request**: `auto-push: false`; the action compares against the latest entry on the data branch + and writes the result to the job summary. `comment-on-alert` is on only when the pull request comes from + this repository, because a fork's token cannot comment. +- **Push to `main`**: both steps run with `auto-push: false` and commit to the local data branch (the second + with `skip-fetch-gh-pages: true`, because fetching the branch again would be refused as a non-fast-forward); + the job then squashes the two commits, adds `latest/` and pushes one commit. The allocation step fails only + on a pull request: on `main` the increase was reviewed, and the new values become the baseline. +- **No data branch yet**: the first push to `main` creates `benchmark-data` with one README commit through the + GitHub API, because the action requires the branch to exist; a pull request before that skips the + comparison with a notice. +- An allocation baseline of 0 bytes: the action computes the ratio against 0; the conversion script writes + allocated bytes plus 1 so a 0 -> N change becomes an alert instead of a division by zero (spec: "a + benchmark that allocated nothing on `main` SHALL fail on any allocation"). + +Alternatives: a hand-written comparer (new tool to maintain); failing on slower times (the +speed of shared CI machines varies, so the check would fail at random); BenchmarkDotNet's own +`--statisticalTest` against a baseline run (needs both versions run on the same machine, which is left out). + +### Accepting a deliberate allocation increase with a label + +The "Allocations" step sets +`fail-on-alert: ${{ !contains(github.event.pull_request.labels.*.name, 'accept-allocation-increase') }}`, so +a labeled pull request still gets the alert in the job summary (and the comment) but passes. The workflow's +`pull_request` trigger lists `types: [opened, synchronize, reopened, labeled, unlabeled]`, so adding or +removing the label re-runs the check. Nothing else is needed: the merged push to `main` records the higher +values, which become the baseline. + +- Only people with triage access or more can label a pull request, so a contributor cannot accept their own + increase; a maintainer does it after reading the alert. +- `labeled` and `unlabeled` re-run the whole CI workflow, not only the benchmark job; the existing + `concurrency` group cancels the run in progress. Accepted: labeling is rare. + +Alternatives: a separate workflow on `labeled` only (a second place that runs the benchmarks); a commit +message marker (contributors could set it themselves); a hand-edited baseline file (has to be maintained). + +### Three parallel benchmark jobs and one report job, on windows-latest + +``` + benchmark-changes --> benchmarks (matrix: Core | Avalonia | WPF) --> benchmark-report + run? yes/no each: restore, build, run, upload JSON merge, two action steps, + latest/ on main +``` + +- **`benchmark-changes`**: a small job that compares the changed files with the pull request's base (or, + on a push, with the previous commit of `main`) using `git diff --name-only`, and outputs whether + anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, + `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`. The other two jobs run only when it + did (spec: "Change without code is not benchmarked"). A skipped job counts as passed for a required check, + so documentation pull requests are not blocked. +- **`benchmarks`**: a three-entry matrix, one suite per entry, all on `windows-latest`. Each builds and runs + its own project and uploads its BenchmarkDotNet artifacts. The wall-clock time is the slowest suite's, + expected to be WPF. One runner type keeps one comparable history. `timeout-minutes: 15`, so a hanging + benchmark cannot hold up CI for long. +- **`benchmark-report`**: needs the matrix, downloads the three JSON reports, runs + `scripts/convert-benchmark-results.ps1` and the two `github-action-benchmark` steps, and on `main` writes + `latest/` to the data branch. Only this job has `contents: write` (push to the data branch) and + `pull-requests: write` (comments); the workflow default stays `contents: read`. + +All of this runs in parallel to the build jobs, so it lengthens CI only if it takes longer than the slowest +of them. + +Alternatives: one job running the three suites one after the other (the wall-clock time is their sum); a +third-party action to detect changed paths (`git diff` does the same without a new dependency). + +### Publishing through the data branch + +The `benchmark-data` branch holds the action's history (`history/data.js` and its chart `index.html`) and, +written by the job on `main` in the same push, the latest BenchmarkDotNet HTML and markdown reports under +`latest/`, one per class, with an `index.html` that links them. `deploy-docs` builds the Pages content from two sources: + +``` + package-and-docs --(artifact: docs-site)--+ + +--> deploy-docs: site + benchmarks/ --> Pages + benchmark-data branch (latest/, history/) -+ +``` + +- `package-and-docs` uploads the DocFX site as an ordinary artifact instead of the Pages artifact. +- `deploy-docs` (main only) downloads it, checks out `benchmark-data` into `benchmarks/` inside the site, + then runs `upload-pages-artifact` and `deploy-pages`. It needs `package-and-docs` and `benchmark-report` + with `if: always() && needs.package-and-docs.result == 'success'`, so a failed or skipped benchmark run + leaves the last successful results on the site (spec: "Benchmark run fails on main"). +- A short page, `docs/guides/benchmarks.md`: a table of each benchmark and its baseline, one sentence on + the Ratio and Allocated columns, and links to + `https://rent-a-developer.github.io/ResXLocalization/benchmarks/latest/` and `.../benchmarks/history/` by + absolute URL, because those files are not DocFX content and relative links would fail + `--warningsAsErrors`. It joins `docs/toc.yml` under Guides. No worked examples: the results answer whether + the library is fast enough by themselves. + +Alternative: commit result markdown into `docs/` from CI (a bot commit on `main` per push). + +## Risks / Trade-offs + +- [The times measured on shared CI machines vary by 10-30% between runs] -> a slower time only raises an + alert, at 150%; only more allocations fail the check. +- [Allocation numbers of UI benchmarks include small allocations of the UI framework, such as dispatcher + work] -> the 25% tolerance, which also leaves room for a feature that adds a small allocation; if a UI + benchmark still fails and passes at random, move it to a third series that alerts without failing. +- [WPF under BenchmarkDotNet's in-process toolchain: STA thread and dispatcher] -> the first task is a + feasibility check that runs one WPF benchmark end to end before the rest is written. +- [Avalonia headless session under the in-process toolchain] -> the same feasibility check for Avalonia. +- [The two frameworks may expect different change notifications for an indexer binding (`"Item[]"` or + `"Item"`), and Avalonia's compiled bindings must accept an indexer on a `Source={x:Static ...}` source] -> + the feasibility check also switches the culture once on a `TranslationSource` binding in each framework, + and the self-check catches a baseline that does not update. +- [The first run on `main` has no history] -> the action does not alert without a previous entry; the first + pull request after merge compares against it. +- [The benchmarks could still make CI slower than the 10-minute target] -> the estimate is about 2 minutes + of restore and build plus 1-4 minutes of measuring per suite. The feasibility check measures the real time + of one suite; if a suite takes over 10 minutes, the suite is cut further before the rest is written. The + first full CI run confirms it. +- [The 10,000-element scaling benchmarks are slow, mostly in WPF] -> those classes run one invocation per + iteration (`InvocationCount = 1`, needed anyway for `[IterationSetup]`). +- [Short iterations make the times vary more between runs] -> accepted: times only alert, at 150%, and + allocations are not affected. +- [A pull request that changes a benchmark's name loses its history] -> accepted; the renamed benchmark is + treated as new and does not fail. +- [The data branch grows with each push to `main`] -> the action keeps at most `max-items-in-chart` entries + (set to 200). + +## Migration Plan + +1. Open the pull request; once its run has passed, archive this change in the same branch, then merge the + projects, the job and the docs page. +2. The first push to `main` creates the `benchmark-data` branch and the first entry. +3. Pull requests compare against it from then on. What only `main` can show - the data branch, the published + pages, a comparison - is a checklist in the pull request's description, checked after merge, because the + change is archived by then. + +Rollback: remove the job and restore the Pages artifact upload in `package-and-docs`; the branch can stay. diff --git a/openspec/changes/add-benchmarks/proposal.md b/openspec/changes/add-benchmarks/proposal.md new file mode 100644 index 0000000..0c2fba4 --- /dev/null +++ b/openspec/changes/add-benchmarks/proposal.md @@ -0,0 +1,71 @@ +# Proposal + +## Why + +ResXLocalization has no performance measurements. Nothing catches a change that makes lookups slower or +allocate more, and consumers cannot see what the library costs them: whether it makes their app slow, and how +it behaves with thousands of localized UI elements and a live culture switch. A small benchmark suite in CI +answers both questions without turning into a performance study. + +## What Changes + +- A BenchmarkDotNet suite under a new `benchmarks/` folder that measures the library's main features against + the code a consumer would write without it - the ResX designer class and hand-written code over it: + - Code lookups through `Localizer.Current.Get` (typed key, scoped key, search-all across several registered + resource managers, enum), with and without `MessageArguments`, against the designer class's property and + `string.Format` over it. + - ICU MessageFormat through `ResXLocalization.MessageFormat` (one message with named arguments), against calling the + MessageFormat library directly. + - XAML lookups for Avalonia (headless) and WPF: the cost per element of each extension form + (`LocalizeExtension`, with `Argument` children, `LocalizeEnumExtension`), measured on 100 elements, + against `{x:Static}` of the designer class. Every UI benchmark is a XAML view compiled by the build and + shown in a window, as in a consumer app. + - Scaling with plain `LocalizeExtension`: opening a view with N localized elements, and switching the + culture with N live bindings, for N = 100 and 10,000, against `{x:Static}` and a hand-written + localization source with an indexer. +- CI jobs that run the suite, in under 10 minutes and in parallel with the existing build jobs, on every pull + request and every push to `main` that changes code, fail a pull request when + a benchmark allocates more than 125% of its bytes on `main`, and reports a time regression above a + threshold without failing. A maintainer can accept a deliberate allocation increase for one pull request + with the label `accept-allocation-increase`, which turns that failure into an alert. +- A published benchmark report on the existing GitHub Pages documentation site: the latest results with + runner details, and the history of every benchmark over the commits on `main`. +- A documentation page that explains what the numbers mean for an application. +- No other localization library is benchmarked; a comparison with one can be added later. +- Benchmarks run on the JIT on .NET 10 only; Native AOT is not benchmarked. + +No shipped package, public API or runtime behavior changes. + +## Capabilities + +### New Capabilities + +- `performance-benchmarks`: the repository's benchmark suite, what it measures and against which baselines, + how CI detects performance regressions with it, and how its results are published. + +### Modified Capabilities + +None. No spec exists yet, and no shipped behavior changes. + +## Impact + +- New projects, one per suite: `benchmarks/ResXLocalization.Core.Benchmarks` (Core and MessageFormat), + `benchmarks/ResXLocalization.Avalonia.Benchmarks` and + `benchmarks/ResXLocalization.WPF.Benchmarks` (Windows only), plus a `benchmarks/Directory.Build.props` and + the code the three share. +- `ResXLocalization.slnx` and `ResXLocalization.NonWindows.slnf` gain the benchmark projects. +- `Directory.Packages.props` gains `BenchmarkDotNet`, referenced only by the benchmark projects, so no + package's dependencies change. +- `.github/workflows/ci.yml` gains a benchmark job, and the documentation deployment includes its results. + Results are stored in two places: + - **CI artifacts** of every run, pull requests and `main`: the raw BenchmarkDotNet reports (JSON, HTML, + markdown, logs), for inspecting that one run. + - **The `benchmark-data` branch**, written only by pushes to `main`, one commit per push: `history/data.js` + with the time and allocation results of the last 200 commits on `main` (the baseline pull requests + compare against, and the data of the history chart), the chart page `history/index.html`, and the HTML + and markdown reports of the latest run under `latest/`. The branch holds no code and is never merged; + the documentation deployment copies it into the site. Writing it needs `contents: write` for the + benchmark job. +- `docs/` gains a benchmarks page and a table-of-contents entry. +- The third-party GitHub Action `benchmark-action/github-action-benchmark` becomes a CI dependency, pinned by + commit SHA like the existing actions. diff --git a/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md new file mode 100644 index 0000000..406703a --- /dev/null +++ b/openspec/changes/add-benchmarks/specs/performance-benchmarks/spec.md @@ -0,0 +1,247 @@ +# Spec Delta + +## Purpose + +Measures what ResXLocalization costs an application and detects performance regressions in the library, so +maintainers see a slower change before it ships and consumers can judge the overhead before they adopt it. + +## ADDED Requirements + +### Requirement: Baselines are the code an application writes without the library +Every benchmark SHALL be reported as a ratio to a baseline that does the same work without ResXLocalization: +the ResX designer class's property, hand-written code over it, or a direct call of the MessageFormat library. +No other localization library SHALL be benchmarked. + +#### Scenario: Every result has a baseline +- **WHEN** the suite runs +- **THEN** every result table contains one baseline row that uses no ResXLocalization API, and every other + row shows its ratio to that row + +#### Scenario: Baseline and library produce the same text +- **WHEN** a benchmark is checked before it is measured +- **THEN** its baseline and its ResXLocalization rows produce the same text for the same key, culture and + arguments + +### Requirement: Code lookups are benchmarked against the ResX designer class +The suite SHALL measure `Localizer.Current.Get` for a typed key, a scoped key, a search-all key and an enum +value, each with mean time and allocated bytes, against the designer class's property for the same key, read +in the same culture. + +#### Scenario: Typed key lookup reports its overhead +- **WHEN** the code lookup benchmarks run +- **THEN** the results contain a designer-property baseline row and a typed-key `Get` row with the same key + and culture, and the `Get` row shows its ratio to the baseline and its allocated bytes + +### Requirement: Search-all lookups are benchmarked by the number of registered resource managers +The suite SHALL measure a search-all lookup whose key exists only in the last registered resource manager, +for 1, 5 and 20 registered resource managers, against hand-written code that searches the same resource +managers, in the same order and culture, until one contains the key. + +#### Scenario: Lookup cost grows with the registered managers +- **WHEN** the search-all benchmark runs +- **THEN** the results contain one row per manager count (1, 5 and 20), each finding the key in the last + registered manager + +#### Scenario: The baseline searches the same managers +- **WHEN** the search-all benchmark runs with a manager count +- **THEN** its baseline row searches the same resource manager instances in the same order and culture, so + the ratio shows only what ResXLocalization adds to the search + +#### Scenario: Managers without the key are searched in the lookup culture +- **WHEN** the search-all benchmark looks up a key in a specific culture, such as `de-DE` +- **THEN** each registered manager that does not contain the key is searched in that culture, including its + culture fallback chain, before the next manager is tried, as in an application + +### Requirement: Format arguments in code are benchmarked +The suite SHALL measure `Get` with positional arguments (`MessageArguments.FromValues`), named arguments +(`MessageArguments.FromTuples`) and object arguments (`MessageArguments.FromObject`), against +`string.Format` over the designer class's property with the same culture and arguments. + +#### Scenario: Each argument form reports its overhead +- **WHEN** the argument benchmarks run +- **THEN** the results contain one row per argument form and a `string.Format` baseline row with the same + pattern, culture and argument values + +### Requirement: ICU MessageFormat is benchmarked against the MessageFormat library +The suite SHALL measure one lookup formatted by the ICU MessageFormat formatter, for a message with two +named arguments, against formatting the designer class's property directly with a formatter of the +MessageFormat library that has the library's default settings, with the same culture and argument values. +The overhead does not depend on which ICU features a message uses, so no other message is measured. + +#### Scenario: One message with named arguments is measured +- **WHEN** the MessageFormat benchmarks run +- **THEN** the results contain a direct MessageFormat library baseline row and one ResXLocalization row for + the same message with two named arguments, and both produce the same text + +#### Scenario: The overhead of the adapter is visible +- **WHEN** the MessageFormat benchmarks run +- **THEN** each ResXLocalization row shows its ratio to the direct MessageFormat library call for the same + message + +### Requirement: UI benchmarks use XAML compiled by the build, as a consumer app does +Every UI benchmark, its baselines included, SHALL write the markup extensions, bindings and `{x:Static}` in +XAML views that the project's build compiles with the UI framework's own tooling, SHALL create those views +as an application does, and SHALL measure until the view is the content of a window and its texts are set. +Rendering SHALL NOT be part of the measurement. + +#### Scenario: A benchmark row is a compiled view +- **WHEN** a UI benchmark runs +- **THEN** the measured operation creates a view defined in a `.axaml` or `.xaml` file of the benchmark + project, makes it the content of a window, and ends when the view's texts are set + +#### Scenario: XAML is not parsed at runtime +- **WHEN** a UI benchmark creates a view +- **THEN** the view's markup was compiled by the build, and no XAML is parsed from a string or file at run + time + +### Requirement: Avalonia benchmarks run on the headless platform +The Avalonia suite SHALL run on Avalonia's headless platform, which keeps Avalonia's controls, XAML, +property system and bindings and replaces only the operating system window, input and rendering. + +#### Scenario: Avalonia suite runs without a desktop +- **WHEN** the Avalonia suite runs on a CI machine +- **THEN** it opens no operating system window and needs no display + +### Requirement: Each extension form is benchmarked for Avalonia and WPF +For each UI framework, the suite SHALL measure opening 100 text elements that all use one form - +`LocalizeExtension` with a typed key, `LocalizeExtension` with two `Argument` children, or +`LocalizeEnumExtension` - from 10-element views used 10 times in one window, against the same layout with +`{x:Static}` of the designer class's property. + +#### Scenario: Each form reports its cost per element +- **WHEN** the forms benchmark runs for a UI framework +- **THEN** the results contain a `{x:Static}` baseline row and one row per form, each for 100 elements, and + every element shows its expected localized text + +### Requirement: Scaling with many localized elements is benchmarked +For each UI framework, the suite SHALL measure opening a view with N localized text elements, and switching +the culture while N localized text elements are live, for N = 100 and 10,000, reporting time and +allocated bytes. The N elements SHALL use `LocalizeExtension` with a typed key and SHALL come from a +compiled view with 10 text elements, used N / 10 times in one window. + +#### Scenario: Culture switch updates every element +- **WHEN** the culture-switch benchmark runs with N live elements +- **THEN** the measurement ends only after all N elements show the text of the new culture + +#### Scenario: Results are reported per element count +- **WHEN** the scaling benchmarks run +- **THEN** the results contain one row per N for opening a view and one row per N for a culture switch + +### Requirement: Scaling benchmarks have library-free baselines +Opening a view SHALL be measured against the same views with `{x:Static}`, and a culture switch against the +same views with every element bound in XAML to a hand-written localization source: a singleton with a string indexer over the designer class's +resource manager that raises a change notification for its indexer when its culture changes. + +#### Scenario: Opening a view is compared with static texts +- **WHEN** the open-view benchmark runs with N elements +- **THEN** the results contain a `{x:Static}` baseline row and a `LocalizeExtension` row for the same N + +#### Scenario: Culture switch is compared with a hand-written source +- **WHEN** the culture-switch benchmark runs with N live elements +- **THEN** the results contain a hand-written-source baseline row and a `LocalizeExtension` row for the same + N, and both end with all N elements showing the text of the new culture + +### Requirement: Benchmarks verify their own results +Every benchmark SHALL check, once before it is measured, that its operation produces the expected text, and +SHALL stop the run with an error when it does not. + +#### Scenario: A broken lookup does not produce a number +- **WHEN** a benchmarked lookup returns the lookup failure text instead of the expected text +- **THEN** the benchmark run fails with an error that names the benchmark, and no result is reported for it + +### Requirement: CI runs the suite on pull requests and on main +CI SHALL run the full suite on every pull request to `main` and on every push to `main` that changes code, +build configuration or the benchmarks, on one fixed runner type and .NET 10 on the JIT. From the start of +the benchmark jobs to the published comparison, the run SHALL take under 10 minutes of wall-clock time. + +#### Scenario: Pull request runs the suite +- **WHEN** a pull request to `main` that changes code is opened or updated +- **THEN** CI runs every benchmark, attaches the raw results to the run, and the benchmark comparison is + reported within 10 minutes of the benchmark jobs starting + +#### Scenario: Change without code is not benchmarked +- **WHEN** a pull request or a push to `main` changes only files outside the library sources, the benchmarks, + the build configuration and the CI workflow, such as documentation +- **THEN** the benchmarks do not run, and the benchmark check does not block the pull request + +### Requirement: Measurements run on optimized code +Every benchmark SHALL be measured only after warmup iterations that let the JIT compile its code fully +optimized, so allocations that optimized code avoids, such as objects the JIT allocates on the stack, are +not counted. + +#### Scenario: Allocations match an application's +- **WHEN** a benchmark's allocations are measured +- **THEN** they are measured after its warmup iterations, on the same optimized code a long-running + application runs + +### Requirement: CI fails a pull request whose benchmarks allocate more +CI SHALL fail a pull request when any benchmark allocates more than 125% of the bytes per operation of the +same benchmark in the latest result from `main`. A benchmark that allocated nothing on `main` SHALL fail on +any allocation. A benchmark with no result on `main` SHALL NOT fail the check. + +#### Scenario: Extra allocation fails the check +- **WHEN** a pull request makes a benchmark allocate more than 125% of its bytes per operation on `main` +- **THEN** the benchmark check fails and names the benchmark with both allocation values + +#### Scenario: Small growth does not fail the check +- **WHEN** a benchmark's bytes per operation grow by at most 25% over `main`, such as a new feature adding a small allocation +- **THEN** the check does not fail because of that benchmark + +#### Scenario: New benchmark does not fail the check +- **WHEN** a pull request adds a benchmark that has no result on `main` +- **THEN** the check does not fail because of that benchmark + +### Requirement: A label accepts an allocation increase for one pull request +While a pull request carries the label `accept-allocation-increase`, CI SHALL report allocation increases +above the threshold as alerts, as it reports time regressions, instead of failing. Adding or removing the +label SHALL re-run the benchmark check. The label SHALL NOT change the check of any other pull request. + +#### Scenario: Labeled pull request passes with an alert +- **WHEN** a pull request with the label `accept-allocation-increase` makes a benchmark allocate more than + 125% of its bytes per operation on `main` +- **THEN** the benchmark check passes, and the job summary lists the benchmark with both allocation values + +#### Scenario: Label is removed +- **WHEN** the label is removed from a pull request whose benchmarks allocate more than the threshold +- **THEN** the benchmark check runs again and fails + +#### Scenario: Accepted increase becomes the new baseline +- **WHEN** a labeled pull request is merged and its push to `main` records new results +- **THEN** later pull requests compare against the increased allocation values + +### Requirement: CI reports time regressions without failing +CI SHALL report every benchmark whose mean time exceeds 150% of the latest result from `main`, in the run's +job summary and, where the run may comment, on the pull request. A time regression alone SHALL NOT fail the +check. + +#### Scenario: Slower benchmark is reported +- **WHEN** a pull request makes a benchmark take more than 150% of its time on `main` +- **THEN** the job summary lists the benchmark with both times, and the check still passes if no allocation + increased + +#### Scenario: Pull request from a fork +- **WHEN** the pull request comes from a fork and the run cannot comment +- **THEN** the comparison appears in the job summary and the run does not fail because it cannot comment + +### Requirement: Benchmark results are published on the documentation site +Each push to `main` SHALL publish, on the GitHub Pages documentation site, that commit's results as tables +with the runner's hardware and runtime details, and a history chart per benchmark over the commits on +`main`. A failed benchmark run SHALL NOT block the documentation deployment. + +#### Scenario: Consumer reads the latest results +- **WHEN** a push to `main` finishes and the documentation site is deployed +- **THEN** the site has that commit's result tables, with the hardware and runtime they were measured on, and + the history chart, both reachable from the documentation's table of contents + +#### Scenario: Benchmark run fails on main +- **WHEN** the benchmark job fails on a push to `main` +- **THEN** the documentation site is still deployed, with the benchmark results of the last successful run + +### Requirement: The documentation links the results +The documentation SHALL have one short page that links the latest results and the history chart and says, +for each benchmark, what it is compared with, and that the Ratio column is the time relative to that +baseline. + +#### Scenario: Reader finds the results +- **WHEN** a developer opens the benchmarks page from the documentation's table of contents +- **THEN** it links the latest results and the history chart and names each benchmark's baseline diff --git a/openspec/changes/add-benchmarks/tasks.md b/openspec/changes/add-benchmarks/tasks.md new file mode 100644 index 0000000..f8ea110 --- /dev/null +++ b/openspec/changes/add-benchmarks/tasks.md @@ -0,0 +1,77 @@ +# Tasks + +## 1. Project setup and feasibility check + +- [x] 1.1 Add `BenchmarkDotNet` to `Directory.Packages.props` (current stable version) and verify `dotnet restore ResXLocalization.slnx` succeeds +- [x] 1.2 Create `benchmarks/Directory.Build.props` (imports the root props, adds `NoWarn` `CS1574;CS1591` with the same comment as the samples, and compiles `benchmarks/Shared/*.cs` into every benchmark project) and the three console projects `benchmarks/ResXLocalization.Core.Benchmarks` (`$(CurrentTargetFramework)`, Core and MessageFormat), `benchmarks/ResXLocalization.Avalonia.Benchmarks` (`$(CurrentTargetFramework)`, Avalonia) and `benchmarks/ResXLocalization.WPF.Benchmarks` (`$(CurrentWindowsTargetFramework)`, `UseWPF`), each with the library, Core and generator `ProjectReference`s the samples use and a `BenchmarkSwitcher` `Program`; verify all three build in Release +- [x] 1.3 Add the three projects to `ResXLocalization.slnx` under `/04_benchmarks/`, and Core and Avalonia (not WPF) to `ResXLocalization.NonWindows.slnf`; verify `dotnet build ResXLocalization.NonWindows.slnf -c Release` and `dotnet test ResXLocalization.slnx -c Release` still pass and `dotnet test` runs no benchmark +- [x] 1.4 Feasibility check: one Avalonia headless benchmark and one WPF `[STAThread]` benchmark that create a compiled XAML view with a `{l:Localize}` `TextBlock`, make it a window's content (headless window in Avalonia, a real `Window` outside the visible screen area in WPF) and check its text, and switch the culture once on a view whose `TextBlock` binds `[key]` on a minimal `TranslationSource` with `Source={x:Static ...}` (in Avalonia with compiled bindings), in their two projects; verify both run end to end with `dotnet run -c Release -- --filter *Feasibility* --inProcess --warmupCount 3 --iterationCount 3 --iterationTime 100` from each project and report time and allocations, record each project's total run time and confirm it leaves a suite of the planned size well under 10 minutes (otherwise cut the suite in the design before going on), then delete the feasibility check classes + - Note: Result: both ran end to end in about 10 s each. Avalonia's compiled indexer binding ignores `"Item[]"`; its `TranslationSource` raises `"Item"`, WPF's raises `Binding.IndexerName` (`"Item[]"`). + +## 2. Resource data and shared configuration + +- [x] 2.1 Write by hand, once in `benchmarks/Shared/Resources/`, `BenchmarkStrings.resx`, `BenchmarkStrings.de.resx` (every key) and a `BenchmarkStrings.Designer.cs` in the shape `PublicResXFileCodeGenerator` emits, linked into all three projects by `benchmarks/Directory.Build.props` with the designer's base name as `LogicalName`: plain texts, a text with two arguments, one ICU message with two named arguments and the 10 `Enum_Priority_Priority1`-`10` keys; plus, in Core only, one `Filler.resx` with a single other key; verify the generator emits `BenchmarkStringsKeys` in the designer's namespace, the designer properties compile, and every project embeds `RentADeveloper.ResXLocalization.Benchmarks.BenchmarkStrings.resources` and builds a `de` satellite with the `.de` resources +- [x] 2.2 Add the shared `[Config]` to `benchmarks/Shared/` (MemoryDiagnoser; JSON full, GitHub markdown and HTML exporters; one job with `InProcessEmitToolchain`, 3 warmup and 15 measured iterations of 100 ms; `ArtifactsPath` set to `artifacts/benchmarks//` from the repository root) and `[STAThread]` on the WPF project's `Main`, plus the culture setup that sets `CultureInfo.CurrentUICulture` and `Localizer.Current.CurrentCulture` to the same culture; verify a run of a class in each project writes all three report files under `artifacts/benchmarks//results` and no `BenchmarkDotNet.Artifacts` folder, runs in-process (no generated project is built) and reports 3 warmup and 15 measured iterations + - Note: The config adds only the JSON exporter; the GitHub markdown and HTML exporters come from BenchmarkDotNet's default configuration, which every `[Config]` is merged with. `Main` runs every benchmark when it gets no arguments, and returns 1 when nothing ran. 3 measured iterations were first planned; with one operation per iteration in the UI classes the error was often larger than the mean, so the job has 15 and the scaling classes 5. + - Note: `Priority`, `ArgumentValues`, `TranslationSource`, `ExpectedViewTexts` and `BenchmarkCulture` live in `benchmarks/Shared/`; `BenchmarkStrings` is one resource file set in `benchmarks/Shared/Resources/`, linked into every project by `benchmarks/Directory.Build.props` with its designer's base name as `LogicalName`; its designer and `BenchmarkStringsKeys` share the namespace of the shared code, so no project has a global using. `BenchmarkCulture.Use` sets `CurrentUICulture`, `Localizer.Current` and `TranslationSource.Instance` together. +- [x] 2.3 Add the self-check helper to `benchmarks/Shared/` that `[GlobalSetup]` calls to run each benchmark method once and throw on an unexpected text, with the expected texts read from the resx files (German from the `.de` satellite alone, without fallback); verify by temporarily breaking one key that the class fails with an error naming the benchmark and produces no result row + +## 3. Core and MessageFormat benchmarks + +- [x] 3.1 `CodeLookupBenchmarks`: designer property baseline (`ResXDesignerProperty`), `GetTypedKey`, `GetStringKeyScoped`, `GetStringKeySearchAll` and `GetEnum`, in `de-DE`; verify with `--filter *CodeLookup*` that every row has a Ratio and Allocated value +- [x] 3.2 `SearchAllBenchmarks` with `[Params(1, 5, 20)]` registered managers: 0, 4 or 19 separate `ResourceManager` instances over `Filler.resx`, then `BenchmarkStrings`, with the key only in `BenchmarkStrings`, against `HandWrittenSearchAll`, a loop of `GetString(key, culture)` over the same managers in the same order; verify three library rows with rising time and a baseline row for each + - Note: Six rows, not three: each manager count has its baseline row. The baseline was first the designer property, which knows the file: against it, 20 managers measured 65 times the time, nearly all `ResourceManager` misses that any search pays. Against the hand-written search the ratio is 1.03-1.14 and the allocations are identical (about 88 B per missed manager, from `ResourceManager`). +- [x] 3.3 `ArgumentBenchmarks`: `string.Format(culture, BenchmarkStrings., ...)` baseline (`StringFormatDesignerProperty`), `GetWithFromValues`, `GetWithFromTuples`, `GetWithFromObject`; verify four rows and that the self-check passes for each +- [x] 3.4 `MessageFormatBenchmarks`: one ICU message with two named arguments (a plural with `{Count}` and `{Name}`); a baseline that formats the designer property with one `new MessageFormatter()` and the same values and culture, against `Get` with `MessageArguments.FromTuples` and `UseMessageFormat()` installed (`MessageFormatterDirect`, `GetWithMessageFormat`); verify two rows and that the self-check confirms both produce the same text + +## 4. Avalonia benchmarks + +- [x] 4.1 Headless session helper in the Avalonia project, started once in each class's `[GlobalSetup]` and running each body on its dispatcher; verify two classes in one run share the session without errors +- [x] 4.2 Four 10-element `.axaml` views under `Views/`, each using one form for all elements and the 10 keys once (`{x:Static res:BenchmarkStrings.}`, `{l:Localize}` typed key, `{l:Localize}` with two `` children bound to the view's `DataContext`, `{l:LocalizeEnum}`), and `AvaloniaFormsBenchmarks`, one row per view (the `{x:Static}` view is the baseline), each opening 10 instances (100 elements) in a `StackPanel` as the headless window's content; verify four rows and that every element's self-check sees its expected text + - Note: The enum view's 10 elements use the 10 members of the shared `Priority` enum. The forms class has `[WarmupCount(50)]` and the host runs without the tiered JIT's call-counting delay (`benchmarks/Directory.Build.props`), because with fewer warmups or the delay the first row ran partly unoptimized code and measured up to three times as slow as the same operation later in the process. +- [x] 4.3 `TranslationSource` in `benchmarks/Shared/`: singleton with a string indexer over `BenchmarkStrings.ResourceManager` and a `Culture` setter that raises one `PropertyChanged` with the framework's indexer name (`#if WINDOWS`: `"Item[]"` in WPF, `"Item"` in Avalonia); verify a compiled-XAML `TextBlock` bound to `[key]` on it updates after a culture change +- [x] 4.4 Write the 10-element `TranslationSource` `.axaml` view (the 10 keys once) and add `AvaloniaScalingBenchmarks`, reusing the `{l:Localize}` and `{x:Static}` views of 4.2, with `[Params(100, 10000)]`: open a view (N / 10 views in a `StackPanel` as the window's content; baseline the `{x:Static}` views) and culture switch (the views built in `[IterationSetup]`, `InvocationCount = 1`, done when the last element shows the new culture's text; baseline the `TranslationSource` views); verify each operation has a baseline and a library row per N, and the run time of the class + - Note: Every UI class has an `[IterationCleanup]` that empties the window, collects garbage and lets the dispatcher go idle; Avalonia processes pending work only down to `DispatcherPriority.Normal`, above layout, as WPF stops at `DataBind`. The application has the Fluent theme, for parity with WPF's default theme styles, and the culture is set on the BenchmarkDotNet thread because the headless dispatch flows its execution context. + +## 5. WPF benchmarks + +- [x] 5.1 WPF helpers: a `Window` shown with `ShowActivated = false` outside the visible screen area that hosts the measured view, and a dispatcher flush with `DispatcherFrame`; verify a compiled-XAML view with a `{l:Localize}` `TextBlock` shows its text after one flush +- [x] 5.2 Use the shared `TranslationSource` in WPF; verify the same way as 4.3 +- [x] 5.3 The four 10-element `.xaml` views under `Views/` and `WpfFormsBenchmarks`, mirroring 4.2, all `[STAThread]`; verify the same rows as Avalonia and that the self-checks pass + - Note: Mirrors 4.2, but with `[WarmupCount(300)]`: WPF's ReadyToRun code takes a few hundred one-operation iterations to be recompiled to optimized code, and with 50 the first row of the process measured about 2 ms too slow. +- [x] 5.4 The `TranslationSource` `.xaml` view and `WpfScalingBenchmarks`, mirroring 4.4; verify the same rows as Avalonia, and that the whole WPF project runs locally in under 6 minutes + - Note: Mirrors 4.4. The idle cleanup matters most here: without it WPF kept the weak-event listeners of every earlier view, and the 100-element culture switch took about six times as long. The whole WPF project runs in about 35 s. +- [x] 5.5 Run the `ui_parity_reviewer` over the Avalonia and WPF benchmark classes and resolve any gap it reports + - Note: The reviewer found one gap, Avalonia's culture switch running all dispatcher work including layout; fixed. + +## 6. CI integration + +- [x] 6.1 Add `scripts/convert-benchmark-results.ps1` that reads every class's BenchmarkDotNet full JSON report, writes `times.json` (their `Benchmarks` arrays, the first report's host info) and `allocations.json` (`customSmallerIsBetter`, allocated bytes + 1 per benchmark); verify on local results that every benchmark of all three projects appears once in each file, with a number +- [x] 6.2 Add the `benchmark-changes` job to `.github/workflows/ci.yml`: `git diff --name-only` against the pull request's base or the push's previous commit, with the output `run` true when anything changed under `src/`, `benchmarks/`, `build/`, the root `Directory.*` files, `global.json`, `.github/workflows/ci.yml` or `scripts/convert-benchmark-results.ps1`; verify with the lint job +- [x] 6.3 Add the `benchmarks` matrix job (Core, Avalonia, WPF) on `windows-latest`, running only when `benchmark-changes` says so: setup, restore, build and run its project (one report per class, no `--join`), upload its `artifacts/benchmarks//` folder, `timeout-minutes: 15`; verify with `pwsh -File scripts/tidy-code.ps1 -Scope all -Check` and `scripts/verify-line-endings.ps1` +- [x] 6.4 Add the `benchmark-report` job that needs the matrix: download the three reports, merge with `scripts/convert-benchmark-results.ps1`, run `benchmark-action/github-action-benchmark` pinned by SHA twice (`benchmarkdotnet` "Time" at 150% without failing, `customSmallerIsBetter` "Allocations" at 125% failing, `summary-always`, `comment-on-alert` only for same-repository pull requests, `auto-push` only on `main`, branch `benchmark-data`, `max-items-in-chart: 200`), with job-level `contents: write` and `pull-requests: write`; verify with the lint job + - Note: Both action steps run with `auto-push: false`; the second skips the fetch. On `main` a final step squashes their commits, adds `latest/` and pushes once, and creates the data branch through the API on the first run. The allocation step fails only on pull requests. +- [x] 6.7 On `main`, have `benchmark-report` copy every class's HTML and markdown report into `latest/` on the `benchmark-data` branch, with an `index.html` that links them + +## 7. Publishing and documentation + +- [x] 7.1 Change `package-and-docs` to upload the DocFX site as an ordinary artifact, and `deploy-docs` to need `package-and-docs` and `benchmark-report` with `if: always() && needs.package-and-docs.result == 'success'`, download the site, check out `benchmark-data` into `benchmarks/` inside it, then upload and deploy the Pages artifact; verify the workflow passes the lint job +- [x] 7.2 Write a short `docs/guides/benchmarks.md` (a table of each benchmark and its baseline, one sentence on the Ratio and Allocated columns, absolute links to `benchmarks/latest/` and `benchmarks/history/`) and add it to `docs/toc.yml` under Guides; verify `dotnet tool run docfx build build/docfx/docfx.json --warningsAsErrors` passes +- [x] 7.3 Add a "Running the benchmarks" section to `CONTRIBUTING.md` (local command, `--filter`, how CI gates allocations and alerts on time, and when a maintainer applies `accept-allocation-increase`) and the `benchmarks/` row to the project map in `AGENTS.md`; verify the documented command runs as written +- [x] 7.4 Add an `[Unreleased]` entry in `CHANGELOG.md` for the published benchmark results page; verify the changelog has no dated section added + +## 8. Integration check + +- [x] 8.1 Run `pwsh -File scripts/pre-commit-gate.ps1` and verify it passes with the benchmark projects included + +## Manual steps and checks + +Done by a maintainer on GitHub, outside this change's tasks: they need the pull request, its CI run or `main`. + +- Create the `accept-allocation-increase` label in the repository, with a description saying it accepts a reviewed allocation increase for one pull request. +- On this pull request's run: `benchmark-changes` outputs `run=true`; the benchmark jobs and the report job pass within 10 minutes of the benchmark jobs starting; the report job shows the "No benchmark baseline" notice; the results are attached to the run. Locally, the job's `git diff` command against a documentation-only commit gives `run=false`. +- Add a post-merge checklist to the pull request's description: the first push to `main` creates the `benchmark-data` branch, the deployed site serves `benchmarks/latest/` and `benchmarks/history/`, and the next pull request's summary compares against that entry with both series. + +## Workflow follow-up + +- Archive the change in this branch after the pull request's run has passed and before it is merged; the post-merge checklist lives in the pull request, not here. diff --git a/scripts/convert-benchmark-results.ps1 b/scripts/convert-benchmark-results.ps1 new file mode 100644 index 0000000..76d86e4 --- /dev/null +++ b/scripts/convert-benchmark-results.ps1 @@ -0,0 +1,88 @@ +<# +.SYNOPSIS + Merges the BenchmarkDotNet reports of the benchmark projects into the two files the CI benchmark comparison + reads: times.json (mean time) and allocations.json (allocated bytes). + +.DESCRIPTION + benchmark-action/github-action-benchmark reads one BenchmarkDotNet JSON file and compares only the mean + time. Every benchmark class writes its own full JSON report, so this script + + - merges every *-report-full.json below -ReportDirectory into times.json: their Benchmarks arrays one + after the other, and the host information of the first report, for the "Time" series (tool: + benchmarkdotnet); + - writes allocations.json in the action's customSmallerIsBetter format, one entry per benchmark with its + allocated bytes per operation, for the "Allocations" series. + + An allocation is written as its bytes plus 1. The action computes a ratio to the value on main, and a + benchmark that allocated nothing there would otherwise be a division by zero instead of an alert when it + starts to allocate. + + A benchmark name is BenchmarkDotNet's FullName, which includes its class and the namespace of its project, + so the names of the classes and projects cannot collide; the script fails if two benchmarks share one anyway, or if a benchmark has + no result, because a missing row would silently drop out of the comparison. + +.PARAMETER ReportDirectory + The directory searched, recursively, for the BenchmarkDotNet *-report-full.json files. + +.PARAMETER OutputDirectory + The directory times.json and allocations.json are written to. It is created if it does not exist. + +.EXAMPLE + pwsh -File scripts/convert-benchmark-results.ps1 -ReportDirectory artifacts/benchmarks -OutputDirectory artifacts/benchmarks/series +#> +#requires -Version 7.0 +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $ReportDirectory, + + [Parameter(Mandatory = $true)] + [string] $OutputDirectory +) + +$ErrorActionPreference = 'Stop' + +$reportFiles = @( + Get-ChildItem -LiteralPath $ReportDirectory -Recurse -File -Filter '*-report-full.json' | Sort-Object FullName +) + +if ($reportFiles.Count -eq 0) +{ + throw "No BenchmarkDotNet *-report-full.json file was found below $ReportDirectory." +} + +$reports = @($reportFiles | ForEach-Object { Get-Content -LiteralPath $_.FullName -Raw | ConvertFrom-Json }) +$benchmarks = @($reports | ForEach-Object { $_.Benchmarks }) +$allocations = New-Object System.Collections.Generic.List[Object] +$names = New-Object System.Collections.Generic.HashSet[String] + +foreach ($benchmark in $benchmarks) +{ + if (-not $names.Add($benchmark.FullName)) + { + throw "Two benchmarks are named $($benchmark.FullName)." + } + + if ($null -eq $benchmark.Statistics -or $null -eq $benchmark.Memory) + { + throw "$($benchmark.FullName) has no result. Its run failed; see the BenchmarkDotNet log." + } + + $allocations.Add( + [ordered]@{ + name = $benchmark.FullName + unit = 'bytes' + value = [long] $benchmark.Memory.BytesAllocatedPerOperation + 1 + } + ) +} + +$times = $reports[0] | Select-Object -Property * -ExcludeProperty Benchmarks +$times | Add-Member -NotePropertyName Benchmarks -NotePropertyValue $benchmarks + +New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null +$times | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'times.json') -Encoding utf8NoBOM +ConvertTo-Json -InputObject $allocations -Depth 4 | + Set-Content -LiteralPath (Join-Path $OutputDirectory 'allocations.json') -Encoding utf8NoBOM + +Write-Output "Merged $($benchmarks.Count) benchmark(s) from $($reportFiles.Count) report(s) into $OutputDirectory."