diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7b81569..01885ee 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,21 +5,19 @@ on: branches: [main] paths: - "docs/**" - - "schemas/**" + - "src/benchx/schemas/**" - "tools/gen_docs.py" - "zensical.toml" - "README.md" - - "benchx/README.md" - ".github/workflows/docs.yml" pull_request: branches: [main] paths: - "docs/**" - - "schemas/**" + - "src/benchx/schemas/**" - "tools/gen_docs.py" - "zensical.toml" - "README.md" - - "benchx/README.md" - ".github/workflows/docs.yml" concurrency: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d467dc8..0f09a55 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -93,20 +93,20 @@ jobs: - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/benchx/uv.lock + cache-dependency-glob: benchx/uv.lock # The scripts run plain `python`, `bx` and `spin`, and hard-code # lib/python3.12/site-packages, so this is Python 3.12 on the PATH. The `test` # extra is only for pyperf, which the benchmarks need. - name: Install benchx on Python 3.12 - working-directory: benchx/benchx + working-directory: benchx run: | uv python install 3.12 uv sync --extra test --python 3.12 echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Install NumPy's build tools - run: uv pip install --python benchx/benchx/.venv/bin/python spin ninja meson-python cython + run: uv pip install --python benchx/.venv/bin/python spin ninja meson-python cython # The full history and the tags: track.sh checks out v2.4.3 and v2.4.0. - name: Clone NumPy @@ -121,7 +121,7 @@ jobs: mkdir -p "$RUNNER_TEMP/pin" cat > "$RUNNER_TEMP/pin/bx" <> "$GITHUB_PATH" @@ -200,12 +200,12 @@ jobs: - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/benchx/uv.lock + cache-dependency-glob: benchx/uv.lock # The scripts run plain `python` and `bx`. benchx is installed in editable mode, # which is where it reads its schemas from. - name: Install benchx - working-directory: benchx/benchx + working-directory: benchx run: | uv python install 3.13 uv sync --extra test --python 3.13 @@ -261,13 +261,13 @@ jobs: - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/benchx/uv.lock + cache-dependency-glob: benchx/uv.lock # The script runs plain `python` and `bx`, and the workers need pyperf, which the # `test` extra brings. benchx is installed in editable mode, which is where it reads # its schemas from. - name: Install benchx - working-directory: benchx/benchx + working-directory: benchx run: | uv python install 3.13 uv sync --extra test --python 3.13 @@ -317,12 +317,12 @@ jobs: - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/benchx/uv.lock + cache-dependency-glob: benchx/uv.lock # The workers need pyperf, which the `test` extra brings. benchx is installed in # editable mode, which is where it reads its schemas from. - name: Install benchx - working-directory: benchx/benchx + working-directory: benchx run: | uv python install 3.13 uv sync --extra test --python 3.13 @@ -332,8 +332,8 @@ jobs: # build is several GB of CUDA libraries that the runner has no use for. - name: Install NumPy and PyTorch run: | - uv pip install --python benchx/benchx/.venv/bin/python numpy - uv pip install --python benchx/benchx/.venv/bin/python torch --index-url https://download.pytorch.org/whl/cpu + uv pip install --python benchx/.venv/bin/python numpy + uv pip install --python benchx/.venv/bin/python torch --index-url https://download.pytorch.org/whl/cpu - name: Show what is under test run: | @@ -381,14 +381,15 @@ jobs: - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/benchx/uv.lock + cache-dependency-glob: benchx/uv.lock # The script makes two venvs of its own with uv (Python 3.12, the newest with numpy - # 1.26.4 wheels) and installs benchx in both, from the checkout that the benchx of the - # active environment runs from. So this environment needs benchx in editable mode, - # which is also where it reads its schemas from. The venvs bring their own pyperf. + # 1.26.4 wheels) and installs benchx in both, from BENCHX_SRC (set on the run step). + # Left unset, it takes the checkout from the benchx of the active environment and assumes + # that the package sits directly in the project directory, which it does not under src/. + # The venvs bring their own pyperf. - name: Install benchx - working-directory: benchx/benchx + working-directory: benchx run: | uv python install 3.13 uv sync --python 3.13 @@ -403,6 +404,8 @@ jobs: - name: Run the numpy-venvs example working-directory: benchx_examples/examples/numpy-venvs + env: + BENCHX_SRC: ${{ github.workspace }}/benchx run: bash ./run.sh - name: Upload the output diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b00fc31..4369d9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,14 +4,18 @@ on: push: branches: [main] paths: - - "benchx/**" - - "schemas/**" + - "src/**" + - "tests/**" + - "pyproject.toml" + - "uv.lock" - ".github/workflows/**" pull_request: branches: [main] paths: - - "benchx/**" - - "schemas/**" + - "src/**" + - "tests/**" + - "pyproject.toml" + - "uv.lock" - ".github/workflows/**" concurrency: @@ -30,16 +34,13 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.11", "3.13"] - defaults: - run: - working-directory: benchx steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 with: enable-cache: true - cache-dependency-glob: benchx/uv.lock + cache-dependency-glob: uv.lock - name: Install Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} @@ -49,3 +50,28 @@ jobs: - name: Run tests run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra + + # An editable install finds the schemas next to the source, so the matrix above cannot + # tell whether a wheel carries them. This installs the built wheel (not editable) into a + # clean environment, runs `bx` from outside the checkout, and runs the test suite that the + # wheel itself ships (benchx/tests), so it checks the wheel's contents as well as its code. + wheel: + name: wheel / py3.13 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Build the wheel + run: uv build --wheel + + - name: Install the wheel into a clean environment and test it + run: | + uv venv --python 3.13 "$RUNNER_TEMP/venv" + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install "$(ls dist/*.whl)[test]" + # Away from the checkout, so that nothing can import it instead of the installed package. + cd "$RUNNER_TEMP" + bx --help + python -m pytest --pyargs benchx -ra diff --git a/.gitignore b/.gitignore index 6f9c72c..ecc3995 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,12 @@ .idea/ site/ .cache/ +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +dist/ # generated by tools/gen_docs.py docs/index.md -docs/benchx.md docs/schemas/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a0e37db --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# 0.1.0 + +Initial release of `benchx`: most major features are implemented, and the project is +ready for user testing. Please give it a spin on your benchmarking needs and let us +know what is suboptimal or missing! + +The following users contributed to this release (listed alphabetically): + +Maanas Arora, +Evgeni Burovski, +Raúl Cumplido, +Ralf Gommers, +Marco Edward Gorelli, +Rok Mihevc + diff --git a/README.md b/README.md index 3601cc7..e7fd602 100644 --- a/README.md +++ b/README.md @@ -25,22 +25,36 @@ lost. ## Grounded in user stories -To make sure the schema and the components serve real needs, we are collecting -user stories from multiple projects — who benchmarks, -what they do today, where it hurts, what must not break. We surveyed Apache Arrow's +To make sure the schema and the components serve real needs, started by collecting +[user stories](https://benchx-dev.github.io/benchx/user_stories/) from multiple projects +-- who benchmarks, what they do today, where it hurts, what must not break. +We surveyed Apache Arrow's continuous benchmarking setup, NumPy, SciPy, Pandas, and several related projects from the data science/numerical computing ecosystem. The stories drive which components exist and what each must do; the schema must allow all of them. +## BenchX usage and development + +We welcome contributions of all sorts: add a user story, tell us about your use case, try +`benchx` on your workflow/project, open an issue if you found a rough edge, +implement a missing feature. + +- Documentation: +- Issue tracker: +- A collection of worked examples: + + ## Contents - `docs/design/benchmark-result-schema.md` — the benchmark result schema and migration contract (the core design document). - `docs/user_stories/` — user story template and collected stories. -- `schemas` -- JSON schemas for the benchmark result and other components. -- `benchx/` -- an in-progress implementation. -- `benchx/examples/` -- worked examples of benchmarking setup(s). +- `src/benchx/` -- an in-progress implementation, with its tests in `tests/`. +- `src/benchx/schemas/` -- JSON schemas for the benchmark result and other components + (shipped with the package). +- worked examples of benchmarking setup(s) are in + [benchx_examples](https://github.com/benchx-dev/benchx_examples). ## Documentation site @@ -53,14 +67,14 @@ uv run --with-requirements docs/requirements.txt python tools/gen_docs.py uv run --with-requirements docs/requirements.txt zensical serve ``` -`tools/gen_docs.py` generates the landing page, the benchx page, and the +`tools/gen_docs.py` generates the landing page and the schema reference pages into `docs/` (those paths are gitignored); `zensical.toml` configures the site, and `.github/workflows/docs.yml` builds and deploys it. ## Status -Beta. The main goal at this stage is to get the schema and the component -boundaries right. The project is under active development, and we expect to be able to +Beta status: the main features are available and ready for user testing; +the project is under active development, and we expect to be able to start offering API stability guarantees in the near future. ## License diff --git a/benchx/.gitignore b/benchx/.gitignore deleted file mode 100644 index 6e2a668..0000000 --- a/benchx/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.venv/ -__pycache__/ -.pytest_cache/ -.ruff_cache/ -.mypy_cache/ diff --git a/benchx/LICENSE b/benchx/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/benchx/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/benchx/pyproject.toml b/benchx/pyproject.toml deleted file mode 100644 index 00aa548..0000000 --- a/benchx/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[project] -name = "benchx" -version = "0.1.0" -description = "BenchX: a flexible and powerful benchmarking framework" -license = "Apache-2.0" -license-files = ["LICENSE"] -requires-python = ">=3.11" -dependencies = [ - "pyarrow>=19", - # format-nongpl makes `format: uri` and `format: date-time` actually checked. - "jsonschema[format-nongpl]>=4.18", - "jsonpointer>=2.0", - "referencing", - "rfc8785", -] - -[project.optional-dependencies] -test = ["pytest", "pyperf", "asv_runner"] - -[project.scripts] -bx = "benchx.cli:main" - -[build-system] -requires = ["hatchling>=1.26"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["benchx"] diff --git a/benchx/README.md b/docs/benchx.md similarity index 82% rename from benchx/README.md rename to docs/benchx.md index eccc80c..9f3c56c 100644 --- a/benchx/README.md +++ b/docs/benchx.md @@ -1,28 +1,32 @@ -# benchx +# BenchX + +BenchX is a powerful and flexible benchmarking framework which aims to address multiple +usage scenarios, from running one-off in-development benchmarks to tracking performance +over history of large projects. + +The project is grounded in [user stories](https://benchx-dev.github.io/benchx/user_stories/) +collected from experience of multiple projects. We aim to address various pain points +we faced and worked around while working on Apache Arrow, NumPy, SciPy, Pandas +and related projects, so that `benchx` becomes the go-to solution for performance benchmarking +across the community. + +We welcome contributions of all sorts: add a user story, tell us about your use case, try +`benchx` on your workflow/project, open an issue if you found a rough edge, +implement a missing feature. + +- Documentation: +- Issue tracker: +- A collection of worked examples: + + +# CLI API + +At a glance: -An implementation of [`docs/design/prototype-design.md`](../docs/design/prototype-design.md): -the slice in [`prototype-scope.md`](../docs/design/prototype-scope.md), from a -work order to a comparison document. ``` -bx run [--out DIR] [--no-ingest] - # runner + Google Benchmark adapter → result files → local store -bx ingest # result files and work orders → the local store -bx series [--workload … --quantity …] # list series -bx history # one series' points (a query); dirty ones are marked -bx head [-n 10] [--project P] [--workload W] [--json|--table] # the latest results, newest first; dirty ones are marked -bx compare --run KEY --profile revisions|environments --baseline VALUE - [--label NAME] [--results DIR] [--k 3] [--json] - # read mode: compare results already measured -bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) -bx compare BASELINE CONTENDER --profile revisions|environments - [--adapter pyperf|google-benchmark] # default pyperf - --suite NAME [--filter REGEX] [--quantity wall-time] - --rounds R [-e NAME=VALUE ...] [--repetitions N] [--min-time S] - [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] - # measure mode: run both sides, then compare -bx validate … [--kind KIND] # every problem in each document, by schema and rules +# Execute benchmarks: + bx bench TARGET --suite NAME [CASE …] # the single-target form of bx compare: one order, a table [--adapter …] [--filter REGEX] [--quantity …] [-e NAME=VALUE ...] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] [--repetitions N] @@ -33,10 +37,39 @@ bx bench MODULE.py [FUNCTION …] [--tree DIR] [...] # shorthand: Python functions under pyperf → results → store → table bx bench PATH --adapter asv [CASE …] [--filter REGEX] [--tree DIR] [...] # shorthand: an asv benchmark suite, timed by pyperf + +# Comparisons: + +# measure mode: run both sides, then compare +bx compare BASELINE CONTENDER --profile revisions|environments + [--adapter pyperf|google-benchmark] # default pyperf + --suite NAME [--filter REGEX] [--quantity wall-time] + --rounds R [-e NAME=VALUE ...] [--repetitions N] [--min-time S] + [--project P] [--label NAME=BASE,CONTENDER] + [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] + +# read mode: compare results already measured +bx compare --run KEY --profile revisions|environments --baseline VALUE + [--label NAME] [--results DIR] [--k 3] [--json] + +# low-level command: run a handwritten work oreder +bx run [--out DIR] [--no-ingest] + # runner + Google Benchmark adapter → result files → local store + +bx ingest # result files and work orders → the local store +bx series [--workload … --quantity …] # list series +bx history # one series' points (a query); dirty ones are marked +bx head [-n 10] [--project P] [--workload W] [--json|--table] # the latest results, newest first; dirty ones are marked + +bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) +bx validate … [--kind KIND] # every problem in each document, by schema and rules bx config init [--dir DIR] [--adapter …] # write a benchx.conf.json stating a harness's defaults bx config show # what benchx.conf.json and the defaults give, and where each value comes from ``` +# Technical details + + **One project file.** `benchx.conf.json` in the project root (found by walking up from the working directory; its directory is the root) says how the project is measured: the `provider`, the `project` name and `source`, an `adapter` block (`name`, `suite`, @@ -94,7 +127,7 @@ is a runnable walk-through. Environments that cannot share a process, such as two venvs with different numpy versions, are measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments ---label NAME --baseline VALUE` compares the sides ([UC-04](../docs/use-cases/UC-04-cross-env.md)). +--label NAME --baseline VALUE` compares the sides ([UC-04](use-cases/UC-04-cross-env.md)). `examples/numpy-venvs` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) is the worked example, with a shell script that drives it. @@ -184,10 +217,13 @@ every result is pyperf. Results also record the installed project's version ## Run it ```console -$ uv venv && uv pip install -e '.[test]' # editable: schemas are read from ../schemas +$ uv venv && uv pip install -e '.[test]' # editable; a plain `uv pip install .` works too, the schemas ship in the package $ .venv/bin/python -m pytest tests # 344 tests, fake Google Benchmark binary, no compiler ``` +A regular (non-editable) install ships the test suite too, so an installed copy can test itself: +`pip install 'benchx[test]'` followed by `python -m pytest --pyargs benchx`. + The end-to-end examples that build real software are in [benchx_examples](https://github.com/benchx-dev/benchx_examples), and the `e2e` workflow runs them. `examples/arrow/demo/run.sh` there is a walkthrough of both comparison @@ -195,37 +231,3 @@ profiles on a small Google Benchmark suite, with real CMake builds (it needs cma C++ compiler, and network access once). It prints every command before running it, as you would type it, so it doubles as a walkthrough. -## Layout - -``` -benchx/ - core/ reading and checking documents; problems are returned, not raised - loader.py strict JSON: duplicate keys, non-finite and inexact numbers - catalog.py the schemas, by kind and version - validation.py which schema a document declares, and whether it fits - rules.py what the result and work-order schemas cannot express - errors.py every problem, with its schema §5.4 rejection code - hashing.py canonical form, hashes, order reference - snapshot.py what the runner records (benchmark-environments.md §3) - adapters/gbench.py Google Benchmark, driving and translating halves - adapters/pyperf.py pyperf, both halves, for Python functions - adapters/asv.py asv's benchmark format, timed by pyperf (bx bench --adapter asv) - adapters/asv_runtime.py asv's discovery and setup semantics, standalone, copied beside each driver - bench.py bx bench: the MODULE.py and asv shorthands, and the results table - config.py benchx.conf.json: finding it, validating it, picking a value, the template - runner.py one work order in, results out - session.py the session loop: alternate two targets over rounds, then compare - target.py target descriptions: the provider call, the sidecar, the declared text - projection.py projections: roles plus an axis, the identity policy the default one (schema §4.3) - identity.py the identity policy and series points (schema §4.3, §4.5) - parquet.py the store's row: #30's columns plus derived ones - store.py the single-file Parquet store and the ingest contract (schema §5.4) - compare.py run mode over a profile's projection, comparison document (comparator.md) - cli.py bx (compare has a read mode and a measure mode) -../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) -tests/ test_prototype.py: the success criteria and the runner and store - rules, on a fake Google Benchmark binary; test_comparator.py: the - comparator's invariants and method on hand-built results; - test_loader, test_validation, test_rules, test_pipeline, - test_validate: core and `bx validate`, on the schema's examples -``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eea6e24 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "benchx" +version = "0.1.0" +description = "BenchX: a flexible and powerful benchmarking framework" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.11" +dependencies = [ + "pyarrow>=19", + # format-nongpl makes `format: uri` and `format: date-time` actually checked. + "jsonschema[format-nongpl]>=4.18", + "jsonpointer>=2.0", + "referencing", + "rfc8785", +] +readme = "docs/benchx.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python" +] + +[project.urls] +Documentation = "https://benchx-dev.github.io/benchx/benchx/" +Repository = "https://github.com/benchx-dev/benchx" + +[project.optional-dependencies] +test = ["pytest", "pyperf", "asv_runner"] + +[project.scripts] +bx = "benchx.cli:main" + +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/benchx"] + +# The test suite ships inside the package, as benchx/tests, so an installed benchx can test +# itself: `python -m pytest --pyargs benchx`. Not as a top-level `tests` package, which would +# collide with other projects' in site-packages. Editable installs do not get it: their +# package is the source directory, and the checkout has tests/. +[tool.hatch.build.targets.wheel.force-include] +"tests" = "benchx/tests" diff --git a/benchx/benchx/__init__.py b/src/benchx/__init__.py similarity index 100% rename from benchx/benchx/__init__.py rename to src/benchx/__init__.py diff --git a/benchx/benchx/adapters/__init__.py b/src/benchx/adapters/__init__.py similarity index 100% rename from benchx/benchx/adapters/__init__.py rename to src/benchx/adapters/__init__.py diff --git a/benchx/benchx/adapters/asv.py b/src/benchx/adapters/asv.py similarity index 100% rename from benchx/benchx/adapters/asv.py rename to src/benchx/adapters/asv.py diff --git a/benchx/benchx/adapters/asv_runtime.py b/src/benchx/adapters/asv_runtime.py similarity index 100% rename from benchx/benchx/adapters/asv_runtime.py rename to src/benchx/adapters/asv_runtime.py diff --git a/benchx/benchx/adapters/base.py b/src/benchx/adapters/base.py similarity index 100% rename from benchx/benchx/adapters/base.py rename to src/benchx/adapters/base.py diff --git a/benchx/benchx/adapters/benchx_params.py b/src/benchx/adapters/benchx_params.py similarity index 100% rename from benchx/benchx/adapters/benchx_params.py rename to src/benchx/adapters/benchx_params.py diff --git a/benchx/benchx/adapters/gbench.py b/src/benchx/adapters/gbench.py similarity index 100% rename from benchx/benchx/adapters/gbench.py rename to src/benchx/adapters/gbench.py diff --git a/benchx/benchx/adapters/pyperf.py b/src/benchx/adapters/pyperf.py similarity index 100% rename from benchx/benchx/adapters/pyperf.py rename to src/benchx/adapters/pyperf.py diff --git a/benchx/benchx/bench.py b/src/benchx/bench.py similarity index 100% rename from benchx/benchx/bench.py rename to src/benchx/bench.py diff --git a/benchx/benchx/cli.py b/src/benchx/cli.py similarity index 100% rename from benchx/benchx/cli.py rename to src/benchx/cli.py diff --git a/benchx/benchx/collectors.py b/src/benchx/collectors.py similarity index 100% rename from benchx/benchx/collectors.py rename to src/benchx/collectors.py diff --git a/benchx/benchx/compare.py b/src/benchx/compare.py similarity index 100% rename from benchx/benchx/compare.py rename to src/benchx/compare.py diff --git a/benchx/benchx/config.py b/src/benchx/config.py similarity index 100% rename from benchx/benchx/config.py rename to src/benchx/config.py diff --git a/benchx/benchx/core/__init__.py b/src/benchx/core/__init__.py similarity index 100% rename from benchx/benchx/core/__init__.py rename to src/benchx/core/__init__.py diff --git a/benchx/benchx/core/catalog.py b/src/benchx/core/catalog.py similarity index 93% rename from benchx/benchx/core/catalog.py rename to src/benchx/core/catalog.py index b02dc72..b0064c5 100644 --- a/benchx/benchx/core/catalog.py +++ b/src/benchx/core/catalog.py @@ -1,4 +1,4 @@ -"""The schemas in the repository's schemas/ directory, or in $BENCHX_SCHEMAS.""" +"""The schemas shipped inside the package (benchx/schemas), or in $BENCHX_SCHEMAS.""" import json import os @@ -9,8 +9,9 @@ from jsonschema import Draft202012Validator, FormatChecker from referencing import Registry, Resource +# Package data, so an installed copy finds them just as an editable install does. SCHEMAS = Path( - os.environ.get("BENCHX_SCHEMAS") or Path(__file__).resolve().parents[3] / "schemas" + os.environ.get("BENCHX_SCHEMAS") or Path(__file__).resolve().parents[1] / "schemas" ) # The version string each document kind declares in its schema_version. Bump a diff --git a/benchx/benchx/core/errors.py b/src/benchx/core/errors.py similarity index 100% rename from benchx/benchx/core/errors.py rename to src/benchx/core/errors.py diff --git a/benchx/benchx/core/hashing.py b/src/benchx/core/hashing.py similarity index 100% rename from benchx/benchx/core/hashing.py rename to src/benchx/core/hashing.py diff --git a/benchx/benchx/core/loader.py b/src/benchx/core/loader.py similarity index 100% rename from benchx/benchx/core/loader.py rename to src/benchx/core/loader.py diff --git a/benchx/benchx/core/rules.py b/src/benchx/core/rules.py similarity index 100% rename from benchx/benchx/core/rules.py rename to src/benchx/core/rules.py diff --git a/benchx/benchx/core/validation.py b/src/benchx/core/validation.py similarity index 100% rename from benchx/benchx/core/validation.py rename to src/benchx/core/validation.py diff --git a/benchx/benchx/errors.py b/src/benchx/errors.py similarity index 100% rename from benchx/benchx/errors.py rename to src/benchx/errors.py diff --git a/benchx/benchx/harnesses.py b/src/benchx/harnesses.py similarity index 100% rename from benchx/benchx/harnesses.py rename to src/benchx/harnesses.py diff --git a/benchx/benchx/identity.py b/src/benchx/identity.py similarity index 100% rename from benchx/benchx/identity.py rename to src/benchx/identity.py diff --git a/benchx/benchx/orders.py b/src/benchx/orders.py similarity index 100% rename from benchx/benchx/orders.py rename to src/benchx/orders.py diff --git a/benchx/benchx/parquet.py b/src/benchx/parquet.py similarity index 98% rename from benchx/benchx/parquet.py rename to src/benchx/parquet.py index c833b6a..0950b5c 100644 --- a/benchx/benchx/parquet.py +++ b/src/benchx/parquet.py @@ -17,7 +17,7 @@ if _VALIDATOR is None: raise RuntimeError( f"result schema not found under {catalog.SCHEMAS}; " - "run from a benchx checkout or set BENCHX_SCHEMAS" + "reinstall benchx, or set BENCHX_SCHEMAS to a directory of schemas" ) RESULT_SCHEMA = _VALIDATOR.schema diff --git a/benchx/benchx/projection.py b/src/benchx/projection.py similarity index 100% rename from benchx/benchx/projection.py rename to src/benchx/projection.py diff --git a/benchx/benchx/runner.py b/src/benchx/runner.py similarity index 100% rename from benchx/benchx/runner.py rename to src/benchx/runner.py diff --git a/schemas/comparison-document/0.1.0/examples/history.json b/src/benchx/schemas/comparison-document/0.1.0/examples/history.json similarity index 100% rename from schemas/comparison-document/0.1.0/examples/history.json rename to src/benchx/schemas/comparison-document/0.1.0/examples/history.json diff --git a/schemas/comparison-document/0.1.0/examples/run.json b/src/benchx/schemas/comparison-document/0.1.0/examples/run.json similarity index 100% rename from schemas/comparison-document/0.1.0/examples/run.json rename to src/benchx/schemas/comparison-document/0.1.0/examples/run.json diff --git a/schemas/comparison-document/0.1.0/schema.json b/src/benchx/schemas/comparison-document/0.1.0/schema.json similarity index 100% rename from schemas/comparison-document/0.1.0/schema.json rename to src/benchx/schemas/comparison-document/0.1.0/schema.json diff --git a/schemas/config/0.1.0/examples/google-benchmark-recommended.json b/src/benchx/schemas/config/0.1.0/examples/google-benchmark-recommended.json similarity index 100% rename from schemas/config/0.1.0/examples/google-benchmark-recommended.json rename to src/benchx/schemas/config/0.1.0/examples/google-benchmark-recommended.json diff --git a/schemas/config/0.1.0/examples/pyperf-recommended.json b/src/benchx/schemas/config/0.1.0/examples/pyperf-recommended.json similarity index 100% rename from schemas/config/0.1.0/examples/pyperf-recommended.json rename to src/benchx/schemas/config/0.1.0/examples/pyperf-recommended.json diff --git a/schemas/config/0.1.0/schema.json b/src/benchx/schemas/config/0.1.0/schema.json similarity index 100% rename from schemas/config/0.1.0/schema.json rename to src/benchx/schemas/config/0.1.0/schema.json diff --git a/schemas/env-identity/0.1.0/examples/arrow-laptop.json b/src/benchx/schemas/env-identity/0.1.0/examples/arrow-laptop.json similarity index 100% rename from schemas/env-identity/0.1.0/examples/arrow-laptop.json rename to src/benchx/schemas/env-identity/0.1.0/examples/arrow-laptop.json diff --git a/schemas/env-identity/0.1.0/schema.json b/src/benchx/schemas/env-identity/0.1.0/schema.json similarity index 100% rename from schemas/env-identity/0.1.0/schema.json rename to src/benchx/schemas/env-identity/0.1.0/schema.json diff --git a/schemas/environment-policy/0.1.0/examples/laptop-default.json b/src/benchx/schemas/environment-policy/0.1.0/examples/laptop-default.json similarity index 100% rename from schemas/environment-policy/0.1.0/examples/laptop-default.json rename to src/benchx/schemas/environment-policy/0.1.0/examples/laptop-default.json diff --git a/schemas/environment-policy/0.1.0/examples/tuned-node.json b/src/benchx/schemas/environment-policy/0.1.0/examples/tuned-node.json similarity index 100% rename from schemas/environment-policy/0.1.0/examples/tuned-node.json rename to src/benchx/schemas/environment-policy/0.1.0/examples/tuned-node.json diff --git a/schemas/environment-policy/0.1.0/schema.json b/src/benchx/schemas/environment-policy/0.1.0/schema.json similarity index 100% rename from schemas/environment-policy/0.1.0/schema.json rename to src/benchx/schemas/environment-policy/0.1.0/schema.json diff --git a/schemas/identity-policy/0.1.0/examples/arrow.json b/src/benchx/schemas/identity-policy/0.1.0/examples/arrow.json similarity index 100% rename from schemas/identity-policy/0.1.0/examples/arrow.json rename to src/benchx/schemas/identity-policy/0.1.0/examples/arrow.json diff --git a/schemas/identity-policy/0.1.0/examples/default.json b/src/benchx/schemas/identity-policy/0.1.0/examples/default.json similarity index 100% rename from schemas/identity-policy/0.1.0/examples/default.json rename to src/benchx/schemas/identity-policy/0.1.0/examples/default.json diff --git a/schemas/identity-policy/0.1.0/schema.json b/src/benchx/schemas/identity-policy/0.1.0/schema.json similarity index 100% rename from schemas/identity-policy/0.1.0/schema.json rename to src/benchx/schemas/identity-policy/0.1.0/schema.json diff --git a/schemas/measurement-result/0.1.0/examples/adhoc.json b/src/benchx/schemas/measurement-result/0.1.0/examples/adhoc.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/adhoc.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/adhoc.json diff --git a/schemas/measurement-result/0.1.0/examples/arrow-temperature.json b/src/benchx/schemas/measurement-result/0.1.0/examples/arrow-temperature.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/arrow-temperature.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/arrow-temperature.json diff --git a/schemas/measurement-result/0.1.0/examples/arrow.json b/src/benchx/schemas/measurement-result/0.1.0/examples/arrow.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/arrow.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/arrow.json diff --git a/schemas/measurement-result/0.1.0/examples/cupy-matmul-cpu-time.json b/src/benchx/schemas/measurement-result/0.1.0/examples/cupy-matmul-cpu-time.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/cupy-matmul-cpu-time.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/cupy-matmul-cpu-time.json diff --git a/schemas/measurement-result/0.1.0/examples/cupy-matmul-gpu-time.json b/src/benchx/schemas/measurement-result/0.1.0/examples/cupy-matmul-gpu-time.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/cupy-matmul-gpu-time.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/cupy-matmul-gpu-time.json diff --git a/schemas/measurement-result/0.1.0/examples/narwhals-overhead-ratio.json b/src/benchx/schemas/measurement-result/0.1.0/examples/narwhals-overhead-ratio.json similarity index 100% rename from schemas/measurement-result/0.1.0/examples/narwhals-overhead-ratio.json rename to src/benchx/schemas/measurement-result/0.1.0/examples/narwhals-overhead-ratio.json diff --git a/schemas/measurement-result/0.1.0/schema.json b/src/benchx/schemas/measurement-result/0.1.0/schema.json similarity index 100% rename from schemas/measurement-result/0.1.0/schema.json rename to src/benchx/schemas/measurement-result/0.1.0/schema.json diff --git a/schemas/target-description/0.1.0/examples/arrow-hardened.json b/src/benchx/schemas/target-description/0.1.0/examples/arrow-hardened.json similarity index 100% rename from schemas/target-description/0.1.0/examples/arrow-hardened.json rename to src/benchx/schemas/target-description/0.1.0/examples/arrow-hardened.json diff --git a/schemas/target-description/0.1.0/examples/empty-text.json b/src/benchx/schemas/target-description/0.1.0/examples/empty-text.json similarity index 100% rename from schemas/target-description/0.1.0/examples/empty-text.json rename to src/benchx/schemas/target-description/0.1.0/examples/empty-text.json diff --git a/schemas/target-description/0.1.0/examples/minimal.json b/src/benchx/schemas/target-description/0.1.0/examples/minimal.json similarity index 100% rename from schemas/target-description/0.1.0/examples/minimal.json rename to src/benchx/schemas/target-description/0.1.0/examples/minimal.json diff --git a/schemas/target-description/0.1.0/schema.json b/src/benchx/schemas/target-description/0.1.0/schema.json similarity index 100% rename from schemas/target-description/0.1.0/schema.json rename to src/benchx/schemas/target-description/0.1.0/schema.json diff --git a/schemas/target-provider-request/0.1.0/examples/describe.json b/src/benchx/schemas/target-provider-request/0.1.0/examples/describe.json similarity index 100% rename from schemas/target-provider-request/0.1.0/examples/describe.json rename to src/benchx/schemas/target-provider-request/0.1.0/examples/describe.json diff --git a/schemas/target-provider-request/0.1.0/examples/prepare.json b/src/benchx/schemas/target-provider-request/0.1.0/examples/prepare.json similarity index 100% rename from schemas/target-provider-request/0.1.0/examples/prepare.json rename to src/benchx/schemas/target-provider-request/0.1.0/examples/prepare.json diff --git a/schemas/target-provider-request/0.1.0/schema.json b/src/benchx/schemas/target-provider-request/0.1.0/schema.json similarity index 100% rename from schemas/target-provider-request/0.1.0/schema.json rename to src/benchx/schemas/target-provider-request/0.1.0/schema.json diff --git a/schemas/work-order/0.1.0/examples/adhoc-minimal.json b/src/benchx/schemas/work-order/0.1.0/examples/adhoc-minimal.json similarity index 100% rename from schemas/work-order/0.1.0/examples/adhoc-minimal.json rename to src/benchx/schemas/work-order/0.1.0/examples/adhoc-minimal.json diff --git a/schemas/work-order/0.1.0/examples/arrow-laptop.json b/src/benchx/schemas/work-order/0.1.0/examples/arrow-laptop.json similarity index 100% rename from schemas/work-order/0.1.0/examples/arrow-laptop.json rename to src/benchx/schemas/work-order/0.1.0/examples/arrow-laptop.json diff --git a/schemas/work-order/0.1.0/examples/draft.json b/src/benchx/schemas/work-order/0.1.0/examples/draft.json similarity index 100% rename from schemas/work-order/0.1.0/examples/draft.json rename to src/benchx/schemas/work-order/0.1.0/examples/draft.json diff --git a/schemas/work-order/0.1.0/schema.json b/src/benchx/schemas/work-order/0.1.0/schema.json similarity index 100% rename from schemas/work-order/0.1.0/schema.json rename to src/benchx/schemas/work-order/0.1.0/schema.json diff --git a/benchx/benchx/session.py b/src/benchx/session.py similarity index 100% rename from benchx/benchx/session.py rename to src/benchx/session.py diff --git a/benchx/benchx/snapshot.py b/src/benchx/snapshot.py similarity index 100% rename from benchx/benchx/snapshot.py rename to src/benchx/snapshot.py diff --git a/benchx/benchx/store.py b/src/benchx/store.py similarity index 100% rename from benchx/benchx/store.py rename to src/benchx/store.py diff --git a/benchx/benchx/target.py b/src/benchx/target.py similarity index 100% rename from benchx/benchx/target.py rename to src/benchx/target.py diff --git a/benchx/tests/conftest.py b/tests/conftest.py similarity index 100% rename from benchx/tests/conftest.py rename to tests/conftest.py diff --git a/benchx/tests/fake_gbench.py b/tests/fake_gbench.py similarity index 100% rename from benchx/tests/fake_gbench.py rename to tests/fake_gbench.py diff --git a/benchx/tests/fixtures/pyperf-calibrated.json b/tests/fixtures/pyperf-calibrated.json similarity index 100% rename from benchx/tests/fixtures/pyperf-calibrated.json rename to tests/fixtures/pyperf-calibrated.json diff --git a/benchx/tests/fixtures/pyperf-native.json b/tests/fixtures/pyperf-native.json similarity index 100% rename from benchx/tests/fixtures/pyperf-native.json rename to tests/fixtures/pyperf-native.json diff --git a/benchx/tests/test_asv.py b/tests/test_asv.py similarity index 100% rename from benchx/tests/test_asv.py rename to tests/test_asv.py diff --git a/benchx/tests/test_bench.py b/tests/test_bench.py similarity index 100% rename from benchx/tests/test_bench.py rename to tests/test_bench.py diff --git a/benchx/tests/test_bench_harnesses.py b/tests/test_bench_harnesses.py similarity index 100% rename from benchx/tests/test_bench_harnesses.py rename to tests/test_bench_harnesses.py diff --git a/benchx/tests/test_benchmark_source.py b/tests/test_benchmark_source.py similarity index 100% rename from benchx/tests/test_benchmark_source.py rename to tests/test_benchmark_source.py diff --git a/benchx/tests/test_build_import_path.py b/tests/test_build_import_path.py similarity index 100% rename from benchx/tests/test_build_import_path.py rename to tests/test_build_import_path.py diff --git a/benchx/tests/test_collectors.py b/tests/test_collectors.py similarity index 100% rename from benchx/tests/test_collectors.py rename to tests/test_collectors.py diff --git a/benchx/tests/test_comparator.py b/tests/test_comparator.py similarity index 100% rename from benchx/tests/test_comparator.py rename to tests/test_comparator.py diff --git a/benchx/tests/test_compare_harnesses.py b/tests/test_compare_harnesses.py similarity index 100% rename from benchx/tests/test_compare_harnesses.py rename to tests/test_compare_harnesses.py diff --git a/benchx/tests/test_config.py b/tests/test_config.py similarity index 100% rename from benchx/tests/test_config.py rename to tests/test_config.py diff --git a/benchx/tests/test_config_schema.py b/tests/test_config_schema.py similarity index 100% rename from benchx/tests/test_config_schema.py rename to tests/test_config_schema.py diff --git a/benchx/tests/test_discovery_timeout.py b/tests/test_discovery_timeout.py similarity index 100% rename from benchx/tests/test_discovery_timeout.py rename to tests/test_discovery_timeout.py diff --git a/benchx/tests/test_e2e_numpy_revisions.py b/tests/test_e2e_numpy_revisions.py similarity index 99% rename from benchx/tests/test_e2e_numpy_revisions.py rename to tests/test_e2e_numpy_revisions.py index a4f0c1a..a74b441 100644 --- a/benchx/tests/test_e2e_numpy_revisions.py +++ b/tests/test_e2e_numpy_revisions.py @@ -58,7 +58,7 @@ def missing_project_file_entries(): - schema = PACKAGE_ROOT.parent / "schemas" / "config" / "0.1.0" / "schema.json" + schema = Path(benchx.__file__).resolve().parent / "schemas" / "config" / "0.1.0" / "schema.json" present = set(json.loads(schema.read_text())["properties"]) if schema.exists() else set() return PROJECT_FILE_ENTRIES - present diff --git a/benchx/tests/test_harnesses.py b/tests/test_harnesses.py similarity index 100% rename from benchx/tests/test_harnesses.py rename to tests/test_harnesses.py diff --git a/benchx/tests/test_loader.py b/tests/test_loader.py similarity index 100% rename from benchx/tests/test_loader.py rename to tests/test_loader.py diff --git a/benchx/tests/test_orders.py b/tests/test_orders.py similarity index 100% rename from benchx/tests/test_orders.py rename to tests/test_orders.py diff --git a/benchx/tests/test_params.py b/tests/test_params.py similarity index 100% rename from benchx/tests/test_params.py rename to tests/test_params.py diff --git a/benchx/tests/test_pipeline.py b/tests/test_pipeline.py similarity index 100% rename from benchx/tests/test_pipeline.py rename to tests/test_pipeline.py diff --git a/benchx/tests/test_projection.py b/tests/test_projection.py similarity index 100% rename from benchx/tests/test_projection.py rename to tests/test_projection.py diff --git a/benchx/tests/test_prototype.py b/tests/test_prototype.py similarity index 100% rename from benchx/tests/test_prototype.py rename to tests/test_prototype.py diff --git a/benchx/tests/test_pyperf.py b/tests/test_pyperf.py similarity index 100% rename from benchx/tests/test_pyperf.py rename to tests/test_pyperf.py diff --git a/benchx/tests/test_rules.py b/tests/test_rules.py similarity index 100% rename from benchx/tests/test_rules.py rename to tests/test_rules.py diff --git a/benchx/tests/test_session.py b/tests/test_session.py similarity index 100% rename from benchx/tests/test_session.py rename to tests/test_session.py diff --git a/benchx/tests/test_sides.py b/tests/test_sides.py similarity index 100% rename from benchx/tests/test_sides.py rename to tests/test_sides.py diff --git a/benchx/tests/test_target.py b/tests/test_target.py similarity index 100% rename from benchx/tests/test_target.py rename to tests/test_target.py diff --git a/benchx/tests/test_uc04.py b/tests/test_uc04.py similarity index 100% rename from benchx/tests/test_uc04.py rename to tests/test_uc04.py diff --git a/benchx/tests/test_validate.py b/tests/test_validate.py similarity index 100% rename from benchx/tests/test_validate.py rename to tests/test_validate.py diff --git a/benchx/tests/test_validation.py b/tests/test_validation.py similarity index 100% rename from benchx/tests/test_validation.py rename to tests/test_validation.py diff --git a/tools/gen_docs.py b/tools/gen_docs.py index 5fecd53..416951c 100644 --- a/tools/gen_docs.py +++ b/tools/gen_docs.py @@ -4,7 +4,6 @@ It writes, into docs/, files that are gitignored and never committed: - index.md, from the repository README (links to docs/... rewritten); -- benchx.md, from benchx/README.md; - schemas/index.md, listing every schema; - schemas/.md, one reference page per schema (newest version); - schemas///schema.json and examples/*.json, so the raw @@ -25,7 +24,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -SCHEMAS = ROOT / "schemas" +SCHEMAS = ROOT / "src" / "benchx" / "schemas" DOCS = ROOT / "docs" REPO_URL = "https://github.com/benchx-dev/benchx" # keep in step with repo_url in zensical.toml URN = re.compile(r"^urn:benchx:schema:(?P[a-z0-9-]+):(?P[0-9.]+)(?:#(?P.*))?$") @@ -399,7 +398,7 @@ def render(self, repo_url: str | None) -> str: src = f"{self.name}/{self.version}/schema.json" links = [f"[schema.json]({src})"] if repo_url: - links.append(f"[source]({repo_url}/blob/main/schemas/{src})") + links.append(f"[source]({repo_url}/blob/main/src/benchx/schemas/{src})") out.append(f"`{s.get('$id', '')}` · version {self.version} · " + " · ".join(links)) if self.older: out.append("") @@ -492,7 +491,7 @@ def rewrite_readme(text: str, strip_prefix: str) -> str: # --------------------------------------------------------------------------- main -GENERATED = ["index.md", "benchx.md", "schemas"] # paths under docs/, all gitignored +GENERATED = ["index.md", "schemas"] # paths under docs/, all gitignored def main() -> None: @@ -516,11 +515,6 @@ def write(rel: str, content: str | bytes) -> None: # Landing page from the repository README. write("index.md", rewrite_readme((ROOT / "README.md").read_text(), "docs/")) - # The README of the benchx implementation. - readme = ROOT / "benchx" / "README.md" - if readme.exists(): - write("benchx.md", readme.read_text().replace("](../docs/", "](")) - # Schema reference pages and raw documents. pages = discover() write("schemas/index.md", schema_index(pages)) diff --git a/benchx/uv.lock b/uv.lock similarity index 100% rename from benchx/uv.lock rename to uv.lock diff --git a/zensical.toml b/zensical.toml index 8d82006..a3dc99f 100644 --- a/zensical.toml +++ b/zensical.toml @@ -1,11 +1,11 @@ # Documentation site, built with Zensical (https://zensical.org). # -# python tools/gen_docs.py # generate docs/index.md, docs/benchx.md, docs/schemas/ +# python tools/gen_docs.py # generate docs/index.md and docs/schemas/ # zensical serve # preview at http://127.0.0.1:8000 # zensical build --strict --clean # what CI runs # # The generated pages are gitignored; run the generator again after changing -# a schema, the README, or the benchx README. +# a schema or the README. [project] site_name = "benchx" @@ -48,8 +48,9 @@ nav = [ { "Target provider request" = "schemas/target-provider-request.md" }, { "Environment policy" = "schemas/environment-policy.md" }, { "Environment identity" = "schemas/env-identity.md" }, - { "Observed-context record" = "schemas/observed-context-record.md" }, + { "Identity policy" = "schemas/identity-policy.md" }, { "Comparison document" = "schemas/comparison-document.md" }, + { "Project configuration" = "schemas/config.md" }, ] }, { "Use cases" = [ { "Overview" = "use-cases/README.md" },