diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index 3c599e6..bcc891f 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -13,6 +13,8 @@ jobs: - uses: astral-sh/setup-uv@v4 + - uses: extractions/setup-just@v2 + - name: Install dependencies run: uv sync --group dev --extra judge diff --git a/.github/workflows/benchmark-smoke.yml b/.github/workflows/benchmark-smoke.yml index 1f55ee9..1cdbe36 100644 --- a/.github/workflows/benchmark-smoke.yml +++ b/.github/workflows/benchmark-smoke.yml @@ -12,6 +12,8 @@ jobs: - uses: astral-sh/setup-uv@v4 + - uses: extractions/setup-just@v2 + - name: Install dependencies run: uv sync --group dev diff --git a/README.md b/README.md index dffc563..537d29e 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Local override: ```bash uv run bm-bench run retrieval \ - --bm-local-path /Users/phernandez/dev/basicmachines/basic-memory + --bm-local-path /path/to/basic-memory ``` ## Mem0 local requirements diff --git a/docs/benchmarks.md b/docs/benchmarks.md index eae3a3d..59a9066 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -42,8 +42,8 @@ It covers: ### Repositories and paths -- benchmark repo: `/Users/phernandez/dev/basicmachines/basic-memory-benchmarks` -- BM local repo (default in `justfile`): `/Users/phernandez/dev/basicmachines/basic-memory` +- benchmark repo: clone of `basicmachines-co/basic-memory-benchmarks` +- BM local repo: set `BM_LOCAL_PATH` env var (or in `.env`) to your local `basic-memory` checkout ### Environment @@ -53,7 +53,7 @@ It covers: ### One-time setup ```bash -cd /Users/phernandez/dev/basicmachines/basic-memory-benchmarks +cd /path/to/basic-memory-benchmarks just sync ``` @@ -107,7 +107,7 @@ Top-level commands: ### One-command full retrieval run ```bash -cd /Users/phernandez/dev/basicmachines/basic-memory-benchmarks +cd /path/to/basic-memory-benchmarks just bench-full ``` @@ -119,7 +119,7 @@ This runs: ### One-command full retrieval + judge ```bash -cd /Users/phernandez/dev/basicmachines/basic-memory-benchmarks +cd /path/to/basic-memory-benchmarks just bench-full-judge ``` @@ -235,8 +235,8 @@ Use this workflow today to compare BM revisions while keeping benchmark tooling ### Step 1: Create BM worktrees for target refs ```bash -BM_REPO=/Users/phernandez/dev/basicmachines/basic-memory -WT_ROOT=/Users/phernandez/dev/basicmachines/basic-memory-benchmarks/benchmarks/worktrees/basic-memory +BM_REPO=/path/to/basic-memory +WT_ROOT=/path/to/basic-memory-benchmarks/benchmarks/worktrees/basic-memory mkdir -p "$WT_ROOT" @@ -250,7 +250,7 @@ git -C "$BM_REPO" worktree add "$WT_ROOT/current" HEAD ### Step 2: Prepare benchmark datasets once ```bash -cd /Users/phernandez/dev/basicmachines/basic-memory-benchmarks +cd /path/to/basic-memory-benchmarks just sync just bench-prepare-short just bench-prepare-long @@ -377,7 +377,7 @@ Planned command shape: ```bash uv run bm-bench run revision-matrix \ - --bm-repo-path /Users/phernandez/dev/basicmachines/basic-memory \ + --bm-repo-path /path/to/basic-memory \ --revisions pre_fusion=f5a0e942^ \ --revisions fusion=f5a0e942 \ --revisions context_step1=f9b2a075 \ diff --git a/justfile b/justfile index d5ef842..ac99227 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,8 @@ set dotenv-load := true # --- Paths and defaults --- -bm_local_path := "/Users/phernandez/dev/basicmachines/basic-memory" +bm_local_path := env_var_or_default("BM_LOCAL_PATH", "") +bm_local_path_flag := if bm_local_path != "" { "--bm-local-path " + bm_local_path } else { "" } locomo_dataset_path := "benchmarks/datasets/locomo/locomo10.json" locomo_output_dir := "benchmarks/generated/locomo" locomo_c1_output_dir := "benchmarks/generated/locomo-c1" @@ -84,7 +85,7 @@ bench-run-short: --corpus-dir benchmarks/generated/locomo-c1/docs \ --queries-path benchmarks/generated/locomo-c1/queries.quick25.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --allow-provider-skip bench-run-short-strict: @@ -94,7 +95,7 @@ bench-run-short-strict: --corpus-dir benchmarks/generated/locomo-c1/docs \ --queries-path benchmarks/generated/locomo-c1/queries.quick25.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --strict-providers # Long benchmark: full LoCoMo query set @@ -105,7 +106,7 @@ bench-run-long: --corpus-dir benchmarks/generated/locomo/docs \ --queries-path benchmarks/generated/locomo/queries.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --allow-provider-skip bench-run-long-strict: @@ -115,7 +116,7 @@ bench-run-long-strict: --corpus-dir benchmarks/generated/locomo/docs \ --queries-path benchmarks/generated/locomo/queries.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --strict-providers bench-run-bm-local: @@ -125,7 +126,7 @@ bench-run-bm-local: --dataset-path {{locomo_dataset_path}} \ --corpus-dir benchmarks/generated/locomo/docs \ --queries-path benchmarks/generated/locomo/queries.json \ - --bm-local-path {{bm_local_path}} + {{bm_local_path_flag}} bench-run-mem0-local: uv run bm-bench run retrieval \ @@ -143,7 +144,7 @@ bench-run-full: --corpus-dir benchmarks/generated/locomo/docs \ --queries-path benchmarks/generated/locomo/queries.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --allow-provider-skip bench-run-full-judge model="gpt-4o-mini": @@ -153,7 +154,7 @@ bench-run-full-judge model="gpt-4o-mini": --corpus-dir benchmarks/generated/locomo/docs \ --queries-path benchmarks/generated/locomo/queries.json \ --providers bm-local,mem0-local \ - --bm-local-path {{bm_local_path}} \ + {{bm_local_path_flag}} \ --allow-provider-skip \ --judge \ --judge-model "{{model}}" diff --git a/pyproject.toml b/pyproject.toml index ad53140..854c348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ ] requires-python = ">=3.12" dependencies = [ - "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@main", + "basic-memory>=0.20.2", "httpx>=0.28.0", "mcp>=1.23.1", "mem0ai==1.0.0", diff --git a/uv.lock b/uv.lock index c789d29..48fe7c3 100644 --- a/uv.lock +++ b/uv.lock @@ -154,8 +154,8 @@ wheels = [ [[package]] name = "basic-memory" -version = "0.18.2.dev71+0a3f3f07" -source = { git = "https://github.com/basicmachines-co/basic-memory?rev=main#0a3f3f07f8f7e1139f8aa509e1267fbdc808d5b3" } +version = "0.20.2" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, { name = "aiosqlite" }, @@ -196,6 +196,10 @@ dependencies = [ { name = "unidecode" }, { name = "watchfiles" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/b1/e2/3d8cc83327b51bb4467ba3bde74bd307bee29b15ae286663261b131a017d/basic_memory-0.20.2.tar.gz", hash = "sha256:6b408e4ebe4cb0fc0600d774b9d0122bce3fd9fe903a5968d582cab68c6a9eb3", size = 1698360, upload-time = "2026-03-11T04:15:01.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/48/8fe5bee09b8b253a2295e8ba84170aee7438ee3261ba13fb61ec9faf39dd/basic_memory-0.20.2-py3-none-any.whl", hash = "sha256:75ec349073ce03b1bdcc3c0f9199c009940348166f82bf79351dea1be9c1e230", size = 786773, upload-time = "2026-03-11T04:14:58.714Z" }, +] [[package]] name = "basic-memory-benchmarks" @@ -227,7 +231,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "basic-memory", git = "https://github.com/basicmachines-co/basic-memory?rev=main" }, + { name = "basic-memory", specifier = ">=0.20.2" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "mcp", specifier = ">=1.23.1" }, { name = "mem0ai", specifier = "==1.0.0" }, @@ -273,10 +277,16 @@ sdist = { url = "https://files.pythonhosted.org/packages/92/88/b8527e1b00c1811db wheels = [ { url = "https://files.pythonhosted.org/packages/d3/25/79c98ebe12df31548ba4eaf44db11b7cad6b3e7b4203718335620939083c/caio-0.9.25-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fb7ff95af4c31ad3f03179149aab61097a71fd85e05f89b4786de0359dffd044", size = 36983, upload-time = "2025-12-26T15:21:36.075Z" }, { url = "https://files.pythonhosted.org/packages/a3/2b/21288691f16d479945968a0a4f2856818c1c5be56881d51d4dac9b255d26/caio-0.9.25-cp312-cp312-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:97084e4e30dfa598449d874c4d8e0c8d5ea17d2f752ef5e48e150ff9d240cd64", size = 82012, upload-time = "2025-12-26T15:22:20.983Z" }, + { url = "https://files.pythonhosted.org/packages/03/c4/8a1b580875303500a9c12b9e0af58cb82e47f5bcf888c2457742a138273c/caio-0.9.25-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:4fa69eba47e0f041b9d4f336e2ad40740681c43e686b18b191b6c5f4c5544bfb", size = 81502, upload-time = "2026-03-04T22:08:22.381Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/0fe770b8ffc8362c48134d1592d653a81a3d8748d764bec33864db36319d/caio-0.9.25-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:6bebf6f079f1341d19f7386db9b8b1f07e8cc15ae13bfdaff573371ba0575d69", size = 80200, upload-time = "2026-03-04T22:08:23.382Z" }, { url = "https://files.pythonhosted.org/packages/31/57/5e6ff127e6f62c9f15d989560435c642144aa4210882f9494204bc892305/caio-0.9.25-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d6c2a3411af97762a2b03840c3cec2f7f728921ff8adda53d7ea2315a8563451", size = 36979, upload-time = "2025-12-26T15:21:35.484Z" }, { url = "https://files.pythonhosted.org/packages/a3/9f/f21af50e72117eb528c422d4276cbac11fb941b1b812b182e0a9c70d19c5/caio-0.9.25-cp313-cp313-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0998210a4d5cd5cb565b32ccfe4e53d67303f868a76f212e002a8554692870e6", size = 81900, upload-time = "2025-12-26T15:22:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/9c/12/c39ae2a4037cb10ad5eb3578eb4d5f8c1a2575c62bba675f3406b7ef0824/caio-0.9.25-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:1a177d4777141b96f175fe2c37a3d96dec7911ed9ad5f02bac38aaa1c936611f", size = 81523, upload-time = "2026-03-04T22:08:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/22/59/f8f2e950eb4f1a5a3883e198dca514b9d475415cb6cd7b78b9213a0dd45a/caio-0.9.25-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:9ed3cfb28c0e99fec5e208c934e5c157d0866aa9c32aa4dc5e9b6034af6286b7", size = 80243, upload-time = "2026-03-04T22:08:26.449Z" }, { url = "https://files.pythonhosted.org/packages/69/ca/a08fdc7efdcc24e6a6131a93c85be1f204d41c58f474c42b0670af8c016b/caio-0.9.25-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fab6078b9348e883c80a5e14b382e6ad6aabbc4429ca034e76e730cf464269db", size = 36978, upload-time = "2025-12-26T15:21:41.055Z" }, { url = "https://files.pythonhosted.org/packages/5e/6c/d4d24f65e690213c097174d26eda6831f45f4734d9d036d81790a27e7b78/caio-0.9.25-cp314-cp314-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44a6b58e52d488c75cfaa5ecaa404b2b41cc965e6c417e03251e868ecd5b6d77", size = 81832, upload-time = "2025-12-26T15:22:22.757Z" }, + { url = "https://files.pythonhosted.org/packages/87/a4/e534cf7d2d0e8d880e25dd61e8d921ffcfe15bd696734589826f5a2df727/caio-0.9.25-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:628a630eb7fb22381dd8e3c8ab7f59e854b9c806639811fc3f4310c6bd711d79", size = 81565, upload-time = "2026-03-04T22:08:27.483Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ed/bf81aeac1d290017e5e5ac3e880fd56ee15e50a6d0353986799d1bc5cfd5/caio-0.9.25-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:0ba16aa605ccb174665357fc729cf500679c2d94d5f1458a6f0d5ca48f2060a7", size = 80071, upload-time = "2026-03-04T22:08:28.751Z" }, { url = "https://files.pythonhosted.org/packages/86/93/1f76c8d1bafe3b0614e06b2195784a3765bbf7b0a067661af9e2dd47fc33/caio-0.9.25-py3-none-any.whl", hash = "sha256:06c0bb02d6b929119b1cfbe1ca403c768b2013a369e2db46bfa2a5761cf82e40", size = 19087, upload-time = "2025-12-26T15:22:00.221Z" }, ]