diff --git a/embeddings/generate-embeddings.py b/embeddings/generate-embeddings.py index e2cb645..6c7c2e6 100644 --- a/embeddings/generate-embeddings.py +++ b/embeddings/generate-embeddings.py @@ -4,11 +4,17 @@ # "datasets", # "sentence-transformers>=5.0.0", # "torch", +# "torchvision", # multimodal embedders such as embeddinggemma-2 need it to load their processor # "numpy", # "pillow", # "einops", # "huggingface-hub>=1.12", # ] +# +# [tool.hf-jobs] +# flavor = "l4x1" +# timeout = "1h" +# secrets = ["HF_TOKEN"] # /// """ Generate embeddings for a Hugging Face dataset (text OR images) with sentence-transformers, @@ -38,23 +44,26 @@ Benchmarks (20k rows, seq-cap 512): all-MiniLM-L6-v2 ~900 rows/s on an L4 (~$0.24/1M rows); bge-base-en-v1.5 ~120 rows/s. L4 is the cheapest flavor for these encoder models. +On Jobs, the [tool.hf-jobs] header sets the l4x1 flavor, a 1h timeout and the HF_TOKEN secret +(`hf` CLI 1.32+). --flavor and --timeout before the script override it. + Examples: # Text (default). Document convention auto-picked. - hf jobs uv run --flavor l4x1 -s HF_TOKEN generate-embeddings.py \\ + hf jobs uv run generate-embeddings.py \\ stanfordnlp/imdb your-name/imdb-embeddings \\ --column text --model sentence-transformers/all-MiniLM-L6-v2 # e5: docs auto-get "passage: ". (--prompt 'passage: ' would be the explicit form.) - hf jobs uv run --flavor l4x1 -s HF_TOKEN generate-embeddings.py \\ + hf jobs uv run generate-embeddings.py \\ stanfordnlp/imdb your-name/imdb-e5 --model intfloat/multilingual-e5-large # Images (CLIP) — prompts don't apply. - hf jobs uv run --flavor l4x1 -s HF_TOKEN generate-embeddings.py \\ + hf jobs uv run generate-embeddings.py \\ your-name/photos your-name/photos-embeddings \\ --modality image --column image --model clip-ViT-B-32 # Test on a small slice first, keep the output private - hf jobs uv run --flavor l4x1 -s HF_TOKEN generate-embeddings.py \\ + hf jobs uv run generate-embeddings.py \\ stanfordnlp/imdb your-name/imdb-emb --max-samples 100 --private FAN-OUT (multi-job) MODE: diff --git a/maintenance/STATUS.md b/maintenance/STATUS.md index 5f607f5..f166d32 100644 --- a/maintenance/STATUS.md +++ b/maintenance/STATUS.md @@ -3,7 +3,7 @@ Generated by `python3 tools/recipe-status.py --write`. Do not edit by hand. Tiers and rules: [MAINTAINING.md](../MAINTAINING.md). -Tiered: 1 of 50 recipes with a header. +Tiered: 1 of 51 recipes with a header. | Recipe | Tier | Image | Packages | Last smoke test | Result | |---|---|---|---|---|---| @@ -14,6 +14,7 @@ Tiered: 1 of 50 recipes with a header. | `classification/train-classifier.py` | — | default | latest | never | | | `classification/train-gliner2.py` | — | default | latest | never | | | `classification/train-kev.py` | — | default | latest | never | | +| `embeddings/generate-embeddings.py` | — | default | latest | 2026-11-10 | [pass](https://huggingface.co/jobs/davanstrien/6af342969737d9a9e090ef50) | | `ocr/deepseek-ocr-vllm.py` | — | `vllm/vllm-openai:v0.29.0` | latest | never | | | `ocr/deepseek-ocr2-vllm.py` | — | `vllm/vllm-openai:v0.29.0` | latest | never | | | `ocr/dots-mocr.py` | — | `vllm/vllm-openai:v0.30.0` | latest | never | | diff --git a/maintenance/verifications.jsonl b/maintenance/verifications.jsonl index 53a590f..6efa0db 100644 --- a/maintenance/verifications.jsonl +++ b/maintenance/verifications.jsonl @@ -4,3 +4,15 @@ {"date": "2026-11-06", "script": "ocr/ovis-ocr2.py", "commit": "6e37e09", "image": "vllm/vllm-openai:v0.29.0", "exclude_newer": "", "model_revision": "main", "flavor": "a10g-small", "input": "davanstrien/lighton-ocr3-speed-test (private; 3 shuffled pages)", "rows": 3, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6aedd70cc78863f9c6346083", "notes": "datasets<5.1 pin (#179). 3/3 rows, no error rows, text read; row 1 is a blank page (empty for every model)"} {"date": "2026-11-06", "script": "ocr/unlimited-ocr-vllm.py", "commit": "be299b7", "image": "vllm/vllm-openai:unlimited-ocr", "exclude_newer": "", "model_revision": "07dea832e22aefee32ad281d4b80551282e1c168", "flavor": "a10g-small", "input": "davanstrien/lighton-ocr3-speed-test (private; 3 shuffled pages)", "rows": 3, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6aede17e0e0338d1bb817171", "notes": "datasets<5.1 pin and model revision pin (#179); main fails since 2026-10-14 (remote code needs addict, matplotlib). 3/3 rows, no error rows, text read."} {"date": "2026-11-06", "script": "ocr/lighton-ocr2.py", "commit": "5ec2132", "image": "vllm/vllm-openai@sha256:953d3a06d5e64ab582985cd7401289d3abf2a2c14ef2158e9a84313daeec77d7", "exclude_newer": "2026-10-01T00:00:00Z", "model_revision": "be7c41313c30e940fe5571de59e5fabc779e00c8", "flavor": "a10g-small", "input": "davanstrien/lighton-ocr3-speed-test (private; 3 shuffled pages)", "rows": 3, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6aede22ac78863f9c6347831", "notes": "First frozen run: digest, cutoff and revision applied (log shows the revision and vLLM v0.22.1). Same output lengths as the unfrozen run on the same pages. 3/3 rows, no error rows, text read; row 1 is a blank page (empty for every model)"} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "fail", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f7aab34849891ac3b90", "notes": "--model nomic-ai/nomic-embed-text-v1.5. The model's remote code calls get_extended_attention_mask, which the installed transformers does not have."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "fail", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f879737d9a9e090e7f0", "notes": "--model google/embeddinggemma-2. ImportError: the processor needs torchvision. Fixed by adding torchvision to the dependencies."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f6a9737d9a9e090e79e", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model BAAI/bge-small-en-v1.5: 384-d, no prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f6d9737d9a9e090e7b3", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model BAAI/bge-base-en-v1.5: 768-d, no prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f709737d9a9e090e7bc", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model BAAI/bge-m3: 1024-d, no prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f739737d9a9e090e7c2", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model sentence-transformers/all-mpnet-base-v2: 768-d, no prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f779737d9a9e090e7cf", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2: 384-d, no prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f7dab34849891ac3ba5", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model intfloat/multilingual-e5-small: 384-d, prefix `passage: `. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f809737d9a9e090e7df", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model intfloat/multilingual-e5-large: 1024-d, prefix `passage: `. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "fa00985", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af33f849737d9a9e090e7e4", "notes": "First run with the [tool.hf-jobs] header, no job flags. --model Qwen/Qwen3-Embedding-0.6B: 1024-d, no document prefix. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "6f5a831", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af342999737d9a9e090ef52", "notes": "With torchvision. --model google/embeddinggemma-2: 768-d, registered prefix `title: none | text: `. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} +{"date": "2026-11-10", "script": "embeddings/generate-embeddings.py", "commit": "6f5a831", "image": "ghcr.io/astral-sh/uv:python3.12-bookworm", "exclude_newer": "", "model_revision": "main", "flavor": "l4x1", "input": "stanfordnlp/imdb (first 100 train rows)", "rows": 100, "outcome": "pass", "job_url": "https://huggingface.co/jobs/davanstrien/6af342969737d9a9e090ef50", "notes": "With torchvision, header only, default model all-MiniLM-L6-v2: 384-d, no prefix, 880 rows/s. 100/100 rows, unit-length vectors, 100 distinct, no NaN; card names the flavor and the prompt."} diff --git a/tools/build-catalog.py b/tools/build-catalog.py index accbe3d..cab4da2 100755 --- a/tools/build-catalog.py +++ b/tools/build-catalog.py @@ -51,12 +51,27 @@ # Recipes without a [tool.hf-jobs] header whose documented command passed a smoke Job with checked output. # Only these and header recipes are declared on model pages. SMOKE_TESTED_EXAMPLES = { - "embeddings/generate-embeddings.py", "gliner/extract-entities.py", "sam3/detect-objects.py", "sam3/segment-objects.py", } +# Models other than a script's default that ran the recipe in a smoke Job with checked output. +# Each gets its own entry on the page and on its model page, with `--model ` in the command. +TESTED_MODELS = { + "embeddings/generate-embeddings.py": [ + "BAAI/bge-small-en-v1.5", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-m3", + "sentence-transformers/all-mpnet-base-v2", + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "intfloat/multilingual-e5-small", + "intfloat/multilingual-e5-large", + "Qwen/Qwen3-Embedding-0.6B", + "google/embeddinggemma-2", + ], +} + # Left out of the Space until they run again. BROKEN = set() @@ -195,6 +210,16 @@ def add_sample_limit(command, arguments): return f"{command} --max-samples {FIRST_RUN_SAMPLES}" +def command_for_model(command, model): + """Return the command with --model set to this model, before any --max-samples at the end.""" + if re.search(r"--model \S+", command): + return re.sub(r"--model \S+", f"--model {model}", command) + limit = re.search(r" --max-samples \d+$", command) + if limit: + return f"{command[:limit.start()]} --model {model}{limit.group(0)}" + return f"{command} --model {model}" + + def command_problems(recipes): """Return the commands that would not run as pasted: template text, or brackets outside quotes.""" problems = [] @@ -284,7 +309,7 @@ def load_recipes(): models = [model for model in models if model not in NOT_HUB_MODELS] if not models: continue - recipes.append({ + recipe = { "kind": "train" if is_training else "run", "task": TRAINING_TASK if is_training else task, "script": script, @@ -299,7 +324,13 @@ def load_recipes(): "command": command, "summary": summarize(tree), "declared": declared, - }) + } + recipes.append(recipe) + for tested_model in TESTED_MODELS.get(script, []): + has_model_option = any(argument["name"] == "--model" for argument in arguments) + if not command or not has_model_option: + sys.exit(f"TESTED_MODELS: {script} needs a command and a --model option") + recipes.append({**recipe, "models": [tested_model], "command": command_for_model(command, tested_model)}) return recipes