From a3532e98ed22e8f1fab76b085f36cd1cb678ff77 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 6 Jun 2026 13:12:49 -0700 Subject: [PATCH 1/3] feat: add proving-grounds pipeline for testing harmont against real OSS repos Single harmont pipeline that clones 15 real-world OSS repos and runs build/test/lint against each using harmont's toolchain abstractions. Covers Rust (ripgrep, clap, tokio, starship, ruff), Python (flask, django, fastapi), npm (express, vite, svelte), Go (gin, terraform), Ruby (jekyll), and a Go+npm monorepo (grafana). Run: hm run ci -d proving-grounds/ --- docs/plans/2026-06-04-proving-grounds.md | 2234 ++++++++++++++++++++++ proving-grounds/.harmont/pipeline.py | 128 ++ 2 files changed, 2362 insertions(+) create mode 100644 docs/plans/2026-06-04-proving-grounds.md create mode 100644 proving-grounds/.harmont/pipeline.py diff --git a/docs/plans/2026-06-04-proving-grounds.md b/docs/plans/2026-06-04-proving-grounds.md new file mode 100644 index 00000000..e16e8408 --- /dev/null +++ b/docs/plans/2026-06-04-proving-grounds.md @@ -0,0 +1,2234 @@ +# Proving Grounds Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build a test suite of 100 real-world OSS repos whose CI pipelines are rewritten in harmont's DSL, proving harmont can express and execute real-world build pipelines. + +**Architecture:** Two-tier testing. *Compile tier* (fast, no Docker): render each pipeline to IR via `hm render`, validate structure via insta snapshots + structural assertions. *Execution tier* (slow, Docker): clone repo at pinned SHA, run `hm run`, verify exit 0. All fixtures live in `proving-grounds/` at repo root — pipeline.py rewrites checked in, original GHA YAML stored for reference, manifest.toml tracks metadata. Initial batch of 15 repos hand-written; remaining 85 added incrementally. + +**Tech Stack:** Python DSL (harmont package), Rust integration tests (insta), shell scripts (fetch/execute), TOML manifests. + +--- + +## Directory Layout + +``` +proving-grounds/ + manifest.toml # All 100 repos: slug, url, sha, lang, complexity, status + scripts/ + fetch-workflows.py # Download GHA YAML from GitHub API + compile-all.sh # hm render every fixture, report pass/fail + execute.sh # Clone + hm run for a single repo + execute-all.sh # Batch execution (nightly) + repos/ + --/ + metadata.toml # Per-repo: url, sha, language, complexity, notes + workflows/ # Original GHA YAML (reference only) + ci.yml + .harmont/ + pipeline.py # Harmont DSL rewrite + tests/ + compile_fixtures.rs # Rust integration test: render + validate IR + snapshots/ # insta snapshot files +``` + +--- + +### Task 1: Create directory structure and manifest format + +**Files:** +- Create: `proving-grounds/manifest.toml` +- Create: `proving-grounds/repos/.gitkeep` (placeholder) + +**Step 1: Create the directory tree** + +Run: +```bash +mkdir -p proving-grounds/{scripts,repos,tests/snapshots} +``` + +**Step 2: Write the manifest with all 100 repos** + +Create `proving-grounds/manifest.toml`: + +```toml +# Proving Grounds Manifest +# status: "pending" | "compile-only" | "passing" | "failing" | "skip" +# complexity: "simple" | "medium" | "complex" | "extreme" + +# ── SIMPLE (30) ───────────────────────────────────────────────────── +[[repos]] +slug = "BurntSushi--ripgrep" +url = "https://github.com/BurntSushi/ripgrep" +sha = "" # pin after first fetch +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "clippy"] + +[[repos]] +slug = "sharkdp--bat" +url = "https://github.com/sharkdp/bat" +sha = "" +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "clippy"] + +[[repos]] +slug = "sharkdp--fd" +url = "https://github.com/sharkdp/fd" +sha = "" +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "ajeetdsouza--zoxide" +url = "https://github.com/ajeetdsouza/zoxide" +sha = "" +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "clap-rs--clap" +url = "https://github.com/clap-rs/clap" +sha = "" +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "clippy"] + +[[repos]] +slug = "serde-rs--serde" +url = "https://github.com/serde-rs/serde" +sha = "" +language = "rust" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "msrv"] + +[[repos]] +slug = "pallets--flask" +url = "https://github.com/pallets/flask" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "psf--requests" +url = "https://github.com/psf/requests" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "psf--black" +url = "https://github.com/psf/black" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "fuzz"] + +[[repos]] +slug = "python-poetry--poetry" +url = "https://github.com/python-poetry/poetry" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "fastapi--fastapi" +url = "https://github.com/fastapi/fastapi" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test", "typecheck"] + +[[repos]] +slug = "tiangolo--sqlmodel" +url = "https://github.com/tiangolo/sqlmodel" +sha = "" +language = "python" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "expressjs--express" +url = "https://github.com/expressjs/express" +sha = "" +language = "javascript" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "sindresorhus--got" +url = "https://github.com/sindresorhus/got" +sha = "" +language = "typescript" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "chalk--chalk" +url = "https://github.com/chalk/chalk" +sha = "" +language = "javascript" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "prettier--prettier" +url = "https://github.com/prettier/prettier" +sha = "" +language = "javascript" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "gin-gonic--gin" +url = "https://github.com/gin-gonic/gin" +sha = "" +language = "go" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "go-chi--chi" +url = "https://github.com/go-chi/chi" +sha = "" +language = "go" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "stretchr--testify" +url = "https://github.com/stretchr/testify" +sha = "" +language = "go" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "junegunn--fzf" +url = "https://github.com/junegunn/fzf" +sha = "" +language = "go" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "jqlang--jq" +url = "https://github.com/jqlang/jq" +sha = "" +language = "c" +complexity = "simple" +status = "pending" +ci_features = ["build", "test"] + +[[repos]] +slug = "jekyll--jekyll" +url = "https://github.com/jekyll/jekyll" +sha = "" +language = "ruby" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "rails--rails" +url = "https://github.com/rails/rails" +sha = "" +language = "ruby" +complexity = "simple" +status = "pending" +ci_features = ["test"] + +[[repos]] +slug = "google--gson" +url = "https://github.com/google/gson" +sha = "" +language = "java" +complexity = "simple" +status = "pending" +ci_features = ["test"] + +[[repos]] +slug = "square--okhttp" +url = "https://github.com/square/okhttp" +sha = "" +language = "kotlin" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "google--guava" +url = "https://github.com/google/guava" +sha = "" +language = "java" +complexity = "simple" +status = "pending" +ci_features = ["test"] + +[[repos]] +slug = "lodash--lodash" +url = "https://github.com/lodash/lodash" +sha = "" +language = "javascript" +complexity = "simple" +status = "pending" +ci_features = ["lint", "test"] + +[[repos]] +slug = "redis--redis" +url = "https://github.com/redis/redis" +sha = "" +language = "c" +complexity = "simple" +status = "pending" +ci_features = ["build", "test"] + +[[repos]] +slug = "JetBrains--kotlin" +url = "https://github.com/JetBrains/kotlin" +sha = "" +language = "kotlin" +complexity = "simple" +status = "pending" +ci_features = ["build", "test"] + +[[repos]] +slug = "Alamofire--Alamofire" +url = "https://github.com/Alamofire/Alamofire" +sha = "" +language = "swift" +complexity = "simple" +status = "pending" +ci_features = ["build", "test"] + +# ── MEDIUM (30) ───────────────────────────────────────────────────── +[[repos]] +slug = "tokio-rs--tokio" +url = "https://github.com/tokio-rs/tokio" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "multi-crate"] + +[[repos]] +slug = "starship--starship" +url = "https://github.com/starship/starship" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching", "release"] + +[[repos]] +slug = "nushell--nushell" +url = "https://github.com/nushell/nushell" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching", "multi-crate"] + +[[repos]] +slug = "helix-editor--helix" +url = "https://github.com/helix-editor/helix" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "clippy", "caching"] + +[[repos]] +slug = "astral-sh--ruff" +url = "https://github.com/astral-sh/ruff" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching", "benchmark"] + +[[repos]] +slug = "django--django" +url = "https://github.com/django/django" +sha = "" +language = "python" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "db-backends"] + +[[repos]] +slug = "scikit-learn--scikit-learn" +url = "https://github.com/scikit-learn/scikit-learn" +sha = "" +language = "python" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "doc-build"] + +[[repos]] +slug = "pandas-dev--pandas" +url = "https://github.com/pandas-dev/pandas" +sha = "" +language = "python" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "benchmark"] + +[[repos]] +slug = "vercel--next.js" +url = "https://github.com/vercel/next.js" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "turbopack"] + +[[repos]] +slug = "sveltejs--svelte" +url = "https://github.com/sveltejs/svelte" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "multi-package"] + +[[repos]] +slug = "vitejs--vite" +url = "https://github.com/vitejs/vite" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "ecosystem-ci"] + +[[repos]] +slug = "withastro--astro" +url = "https://github.com/withastro/astro" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "multi-package"] + +[[repos]] +slug = "pnpm--pnpm" +url = "https://github.com/pnpm/pnpm" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching"] + +[[repos]] +slug = "denoland--deno" +url = "https://github.com/denoland/deno" +sha = "" +language = "rust" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching", "multi-suite"] + +[[repos]] +slug = "prometheus--prometheus" +url = "https://github.com/prometheus/prometheus" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "docker-build"] + +[[repos]] +slug = "hashicorp--terraform" +url = "https://github.com/hashicorp/terraform" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os", "caching", "acceptance"] + +[[repos]] +slug = "hashicorp--vault" +url = "https://github.com/hashicorp/vault" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "integration"] + +[[repos]] +slug = "etcd-io--etcd" +url = "https://github.com/etcd-io/etcd" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "integration"] + +[[repos]] +slug = "golang--go" +url = "https://github.com/golang/go" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["multi-builder", "caching"] + +[[repos]] +slug = "curl--curl" +url = "https://github.com/curl/curl" +sha = "" +language = "c" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os-compiler", "caching"] + +[[repos]] +slug = "facebook--zstd" +url = "https://github.com/facebook/zstd" +sha = "" +language = "c" +complexity = "medium" +status = "pending" +ci_features = ["matrix-os-compiler", "caching"] + +[[repos]] +slug = "spring-projects--spring-boot" +url = "https://github.com/spring-projects/spring-boot" +sha = "" +language = "java" +complexity = "medium" +status = "pending" +ci_features = ["matrix-jdk", "caching", "integration"] + +[[repos]] +slug = "elastic--elasticsearch" +url = "https://github.com/elastic/elasticsearch" +sha = "" +language = "java" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "integration"] + +[[repos]] +slug = "apache--kafka" +url = "https://github.com/apache/kafka" +sha = "" +language = "java" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "system-tests"] + +[[repos]] +slug = "vapor--vapor" +url = "https://github.com/vapor/vapor" +sha = "" +language = "swift" +complexity = "medium" +status = "pending" +ci_features = ["matrix-swift", "linux-macos"] + +[[repos]] +slug = "storybookjs--storybook" +url = "https://github.com/storybookjs/storybook" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "multi-framework"] + +[[repos]] +slug = "angular--angular" +url = "https://github.com/angular/angular" +sha = "" +language = "typescript" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "bazel"] + +[[repos]] +slug = "pytorch--pytorch" +url = "https://github.com/pytorch/pytorch" +sha = "" +language = "python" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "caching", "cuda", "artifacts"] + +[[repos]] +slug = "goreleaser--goreleaser" +url = "https://github.com/goreleaser/goreleaser" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["matrix", "docker-multi-arch", "release"] + +[[repos]] +slug = "docker--compose" +url = "https://github.com/docker/compose" +sha = "" +language = "go" +complexity = "medium" +status = "pending" +ci_features = ["multi-platform", "docker-in-docker", "release"] + +# ── COMPLEX (25) ──────────────────────────────────────────────────── +[[repos]] +slug = "rust-lang--rust" +url = "https://github.com/rust-lang/rust" +sha = "" +language = "rust" +complexity = "complex" +status = "pending" +ci_features = ["massive-matrix", "cross-compilation", "artifacts", "bors"] + +[[repos]] +slug = "alacritty--alacritty" +url = "https://github.com/alacritty/alacritty" +sha = "" +language = "rust" +complexity = "complex" +status = "pending" +ci_features = ["cross-platform", "release-artifacts", "caching"] + +[[repos]] +slug = "tauri-apps--tauri" +url = "https://github.com/tauri-apps/tauri" +sha = "" +language = "rust" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "release-automation", "docker", "cross-compile"] + +[[repos]] +slug = "zed-industries--zed" +url = "https://github.com/zed-industries/zed" +sha = "" +language = "rust" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "release", "caching", "signing"] + +[[repos]] +slug = "oven-sh--bun" +url = "https://github.com/oven-sh/bun" +sha = "" +language = "zig" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "cross-compilation", "docker", "release"] + +[[repos]] +slug = "huggingface--transformers" +url = "https://github.com/huggingface/transformers" +sha = "" +language = "python" +complexity = "complex" +status = "pending" +ci_features = ["matrix", "gpu-runners", "doc-deploy", "model-tests"] + +[[repos]] +slug = "tensorflow--tensorflow" +url = "https://github.com/tensorflow/tensorflow" +sha = "" +language = "cpp" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "gpu", "docker", "release"] + +[[repos]] +slug = "python--cpython" +url = "https://github.com/python/cpython" +sha = "" +language = "c" +complexity = "complex" +status = "pending" +ci_features = ["matrix-os-compiler", "cross-compilation", "release", "extensive-tests"] + +[[repos]] +slug = "facebook--react" +url = "https://github.com/facebook/react" +sha = "" +language = "javascript" +complexity = "complex" +status = "pending" +ci_features = ["multi-package", "artifacts", "deploy", "canary"] + +[[repos]] +slug = "microsoft--TypeScript" +url = "https://github.com/microsoft/TypeScript" +sha = "" +language = "typescript" +complexity = "complex" +status = "pending" +ci_features = ["matrix", "release", "benchmarks", "artifacts"] + +[[repos]] +slug = "eslint--eslint" +url = "https://github.com/eslint/eslint" +sha = "" +language = "javascript" +complexity = "complex" +status = "pending" +ci_features = ["matrix", "caching", "semantic-release"] + +[[repos]] +slug = "prisma--prisma" +url = "https://github.com/prisma/prisma" +sha = "" +language = "typescript" +complexity = "complex" +status = "pending" +ci_features = ["matrix-db", "docker-test-dbs", "release"] + +[[repos]] +slug = "containerd--containerd" +url = "https://github.com/containerd/containerd" +sha = "" +language = "go" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "cross-compilation", "integration", "release"] + +[[repos]] +slug = "grpc--grpc" +url = "https://github.com/grpc/grpc" +sha = "" +language = "cpp" +complexity = "complex" +status = "pending" +ci_features = ["matrix-languages", "multi-platform", "docker"] + +[[repos]] +slug = "llvm--llvm-project" +url = "https://github.com/llvm/llvm-project" +sha = "" +language = "cpp" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "massive-matrix", "sanitizers", "release"] + +[[repos]] +slug = "FFmpeg--FFmpeg" +url = "https://github.com/FFmpeg/FFmpeg" +sha = "" +language = "c" +complexity = "complex" +status = "pending" +ci_features = ["cross-compilation", "multi-platform", "configure-matrix"] + +[[repos]] +slug = "flutter--flutter" +url = "https://github.com/flutter/flutter" +sha = "" +language = "dart" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "device-testing", "release-channels"] + +[[repos]] +slug = "expo--expo" +url = "https://github.com/expo/expo" +sha = "" +language = "typescript" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform-mobile", "release", "monorepo", "caching"] + +[[repos]] +slug = "grafana--grafana" +url = "https://github.com/grafana/grafana" +sha = "" +language = "go" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "docker", "release", "frontend-backend"] + +[[repos]] +slug = "cockroachdb--cockroach" +url = "https://github.com/cockroachdb/cockroach" +sha = "" +language = "go" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "docker", "release", "integration"] + +[[repos]] +slug = "tikv--tikv" +url = "https://github.com/tikv/tikv" +sha = "" +language = "rust" +complexity = "complex" +status = "pending" +ci_features = ["matrix", "cross-compilation", "integration", "release"] + +[[repos]] +slug = "apache--spark" +url = "https://github.com/apache/spark" +sha = "" +language = "scala" +complexity = "complex" +status = "pending" +ci_features = ["matrix-jdk-scala-hadoop", "docker", "extensive-tests"] + +[[repos]] +slug = "SignalApp--Signal-Android" +url = "https://github.com/SignalApp/Signal-Android" +sha = "" +language = "kotlin" +complexity = "complex" +status = "pending" +ci_features = ["android-build", "signing", "artifacts", "release"] + +[[repos]] +slug = "neovim--neovim" +url = "https://github.com/neovim/neovim" +sha = "" +language = "c" +complexity = "complex" +status = "pending" +ci_features = ["multi-platform", "caching", "release", "functional-tests"] + +[[repos]] +slug = "supabase--supabase" +url = "https://github.com/supabase/supabase" +sha = "" +language = "typescript" +complexity = "complex" +status = "pending" +ci_features = ["monorepo", "docker", "multi-service", "release"] + +# ── EXTREME (15) ──────────────────────────────────────────────────── +[[repos]] +slug = "kubernetes--kubernetes" +url = "https://github.com/kubernetes/kubernetes" +sha = "" +language = "go" +complexity = "extreme" +status = "pending" +ci_features = ["prow", "massive-matrix", "self-hosted", "conditional", "conformance"] + +[[repos]] +slug = "NixOS--nixpkgs" +url = "https://github.com/NixOS/nixpkgs" +sha = "" +language = "nix" +complexity = "extreme" +status = "pending" +ci_features = ["hydra", "80k-packages", "self-hosted", "conditional"] + +[[repos]] +slug = "Homebrew--homebrew-core" +url = "https://github.com/Homebrew/homebrew-core" +sha = "" +language = "ruby" +complexity = "extreme" +status = "pending" +ci_features = ["massive-formula-matrix", "bottle-builds", "self-hosted"] + +[[repos]] +slug = "vercel--turborepo" +url = "https://github.com/vercel/turborepo" +sha = "" +language = "rust" +complexity = "extreme" +status = "pending" +ci_features = ["monorepo", "reusable-workflows", "remote-caching"] + +[[repos]] +slug = "nrwl--nx" +url = "https://github.com/nrwl/nx" +sha = "" +language = "typescript" +complexity = "extreme" +status = "pending" +ci_features = ["monorepo", "affected-ci", "distributed-execution"] + +[[repos]] +slug = "chromium--chromium" +url = "https://github.com/nicedoc/nicedoc.io" +sha = "" +language = "cpp" +complexity = "extreme" +status = "pending" +ci_features = ["luci", "massive-matrix", "self-hosted", "sanitizers"] +notes = "chromium uses LUCI, not GHA - reference only" + +[[repos]] +slug = "microsoft--vscode" +url = "https://github.com/microsoft/vscode" +sha = "" +language = "typescript" +complexity = "extreme" +status = "pending" +ci_features = ["multi-platform", "electron", "release", "self-hosted", "insider-builds"] + +[[repos]] +slug = "facebook--react-native" +url = "https://github.com/facebook/react-native" +sha = "" +language = "javascript" +complexity = "extreme" +status = "pending" +ci_features = ["multi-platform", "self-hosted", "conditional", "hermes"] + +[[repos]] +slug = "rust-lang--cargo" +url = "https://github.com/rust-lang/cargo" +sha = "" +language = "rust" +complexity = "extreme" +status = "pending" +ci_features = ["reusable-workflows", "matrix", "cross-platform", "bors"] + +[[repos]] +slug = "actions--runner" +url = "https://github.com/actions/runner" +sha = "" +language = "csharp" +complexity = "extreme" +status = "pending" +ci_features = ["self-referential", "multi-platform", "release", "reusable-workflows"] + +[[repos]] +slug = "github--codeql" +url = "https://github.com/github/codeql" +sha = "" +language = "codeql" +complexity = "extreme" +status = "pending" +ci_features = ["reusable-workflows", "multi-language-matrix", "composite-actions"] + +[[repos]] +slug = "nixos--nix" +url = "https://github.com/NixOS/nix" +sha = "" +language = "cpp" +complexity = "extreme" +status = "pending" +ci_features = ["multi-platform", "self-hosted", "flake-ci", "docker", "release"] + +[[repos]] +slug = "envoyproxy--envoy" +url = "https://github.com/envoyproxy/envoy" +sha = "" +language = "cpp" +complexity = "extreme" +status = "pending" +ci_features = ["multi-platform", "bazel", "docker", "self-hosted", "sanitizers"] + +[[repos]] +slug = "pulumi--pulumi" +url = "https://github.com/pulumi/pulumi" +sha = "" +language = "go" +complexity = "extreme" +status = "pending" +ci_features = ["monorepo", "multi-language-sdk", "integration", "release", "reusable-workflows"] + +[[repos]] +slug = "hashicorp--nomad" +url = "https://github.com/hashicorp/nomad" +sha = "" +language = "go" +complexity = "extreme" +status = "pending" +ci_features = ["multi-platform", "integration", "self-hosted", "conditional", "release"] +``` + +**Step 3: Commit** + +```bash +git add proving-grounds/ +git commit -m "feat: proving-grounds scaffold with 100-repo manifest" +``` + +--- + +### Task 2: Write the workflow fetch script + +**Files:** +- Create: `proving-grounds/scripts/fetch-workflows.py` + +**Step 1: Write the fetch script** + +```python +#!/usr/bin/env python3 +"""Fetch .github/workflows/ from repos listed in manifest.toml. + +Usage: + python fetch-workflows.py [--token GITHUB_TOKEN] [--slug SLUG] + +Without --slug, fetches all repos with status != "skip". +With --slug, fetches only that repo. +Pins SHA to HEAD of default branch if sha is empty. +""" +import argparse +import json +import os +import sys +import urllib.request +from pathlib import Path + +MANIFEST = Path(__file__).resolve().parent.parent / "manifest.toml" +REPOS_DIR = Path(__file__).resolve().parent.parent / "repos" + + +def load_manifest(): + # Minimal TOML parser for our simple format — avoids external dep + import tomllib + return tomllib.loads(MANIFEST.read_text()) + + +def gh_api(path: str, token: str | None) -> dict: + url = f"https://api.github.com{path}" + req = urllib.request.Request(url) + req.add_header("Accept", "application/vnd.github.v3+json") + if token: + req.add_header("Authorization", f"token {token}") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def gh_raw(owner: str, repo: str, sha: str, path: str, token: str | None) -> bytes: + url = f"https://raw.githubusercontent.com/{owner}/{repo}/{sha}/{path}" + req = urllib.request.Request(url) + if token: + req.add_header("Authorization", f"token {token}") + with urllib.request.urlopen(req) as resp: + return resp.read() + + +def fetch_repo(entry: dict, token: str | None): + slug = entry["slug"] + url = entry["url"] + parts = url.rstrip("/").split("/") + owner, repo = parts[-2], parts[-1] + + sha = entry.get("sha", "") + if not sha: + info = gh_api(f"/repos/{owner}/{repo}", token) + sha = gh_api( + f"/repos/{owner}/{repo}/commits/{info['default_branch']}", token + )["sha"] + print(f" pinned {slug} -> {sha[:12]}") + + repo_dir = REPOS_DIR / slug + wf_dir = repo_dir / "workflows" + wf_dir.mkdir(parents=True, exist_ok=True) + + # Write metadata + meta = repo_dir / "metadata.toml" + meta.write_text( + f'url = "{url}"\n' + f'sha = "{sha}"\n' + f'language = "{entry["language"]}"\n' + f'complexity = "{entry["complexity"]}"\n' + ) + + # Fetch workflow files + try: + contents = gh_api( + f"/repos/{owner}/{repo}/contents/.github/workflows?ref={sha}", token + ) + except Exception as e: + print(f" WARN: no workflows for {slug}: {e}") + return sha + + for item in contents: + if item["name"].endswith((".yml", ".yaml")): + data = gh_raw(owner, repo, sha, item["path"], token) + (wf_dir / item["name"]).write_bytes(data) + print(f" {slug}/workflows/{item['name']}") + + return sha + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--token", default=os.environ.get("GITHUB_TOKEN")) + parser.add_argument("--slug", help="Fetch only this repo slug") + args = parser.parse_args() + + manifest = load_manifest() + repos = manifest.get("repos", []) + + if args.slug: + repos = [r for r in repos if r["slug"] == args.slug] + if not repos: + print(f"Unknown slug: {args.slug}", file=sys.stderr) + sys.exit(1) + + for entry in repos: + if entry.get("status") == "skip": + continue + print(f"Fetching {entry['slug']}...") + try: + fetch_repo(entry, args.token) + except Exception as e: + print(f" ERROR: {e}") + + +if __name__ == "__main__": + main() +``` + +**Step 2: Make executable** + +Run: `chmod +x proving-grounds/scripts/fetch-workflows.py` + +**Step 3: Commit** + +```bash +git add proving-grounds/scripts/fetch-workflows.py +git commit -m "feat: workflow fetch script for proving-grounds" +``` + +--- + +### Task 3: Write compile-tier test script + +**Files:** +- Create: `proving-grounds/scripts/compile-all.sh` + +**Step 1: Write the compile test script** + +This script iterates over all repos that have a `.harmont/pipeline.py`, runs `hm render`, and reports pass/fail. + +```bash +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPOS_DIR="$SCRIPT_DIR/../repos" +HM="${HM_BIN:-hm}" + +pass=0 +fail=0 +skip=0 +failures=() + +for repo_dir in "$REPOS_DIR"/*/; do + slug="$(basename "$repo_dir")" + pipeline="$repo_dir/.harmont/pipeline.py" + + if [[ ! -f "$pipeline" ]]; then + skip=$((skip + 1)) + continue + fi + + if $HM render ci --dir "$repo_dir" > /dev/null 2>&1; then + echo "PASS $slug" + pass=$((pass + 1)) + else + echo "FAIL $slug" + fail=$((fail + 1)) + failures+=("$slug") + fi +done + +echo "" +echo "── Results ──" +echo "pass: $pass fail: $fail skip: $skip" + +if [[ ${#failures[@]} -gt 0 ]]; then + echo "" + echo "Failures:" + for f in "${failures[@]}"; do + echo " - $f" + done + exit 1 +fi +``` + +**Step 2: Make executable** + +Run: `chmod +x proving-grounds/scripts/compile-all.sh` + +**Step 3: Commit** + +```bash +git add proving-grounds/scripts/compile-all.sh +git commit -m "feat: compile-tier test script for proving-grounds" +``` + +--- + +### Task 4: Write execution-tier test scripts + +**Files:** +- Create: `proving-grounds/scripts/execute.sh` +- Create: `proving-grounds/scripts/execute-all.sh` + +**Step 1: Write the single-repo execution script** + +```bash +#!/usr/bin/env bash +set -euo pipefail + +# Usage: execute.sh [--keep] +# Clones repo at pinned SHA, copies .harmont/, runs hm run ci. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPOS_DIR="$SCRIPT_DIR/../repos" +HM="${HM_BIN:-hm}" +CACHE_DIR="${PROVING_GROUNDS_CACHE:-/tmp/proving-grounds}" + +slug="$1" +keep="${2:-}" +repo_dir="$REPOS_DIR/$slug" +meta="$repo_dir/metadata.toml" + +if [[ ! -f "$meta" ]]; then + echo "ERROR: no metadata.toml for $slug" >&2 + exit 1 +fi + +url=$(grep '^url' "$meta" | cut -d'"' -f2) +sha=$(grep '^sha' "$meta" | cut -d'"' -f2) + +if [[ -z "$sha" ]]; then + echo "ERROR: no pinned SHA for $slug" >&2 + exit 1 +fi + +clone_dir="$CACHE_DIR/$slug" + +if [[ -d "$clone_dir/.git" ]]; then + echo "Using cached clone: $clone_dir" + git -C "$clone_dir" checkout "$sha" 2>/dev/null || { + git -C "$clone_dir" fetch origin "$sha" + git -C "$clone_dir" checkout "$sha" + } +else + echo "Cloning $url @ $sha ..." + git clone --depth 1 "$url" "$clone_dir" 2>/dev/null || { + git clone "$url" "$clone_dir" + } + git -C "$clone_dir" checkout "$sha" 2>/dev/null || true +fi + +# Copy harmont pipeline into clone +if [[ -d "$repo_dir/.harmont" ]]; then + cp -r "$repo_dir/.harmont" "$clone_dir/.harmont" +fi + +echo "Running: $HM run ci --dir $clone_dir" +$HM run ci --dir "$clone_dir" +result=$? + +if [[ "$keep" != "--keep" ]]; then + rm -rf "$clone_dir" +fi + +exit $result +``` + +**Step 2: Write the batch execution script** + +```bash +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPOS_DIR="$SCRIPT_DIR/../repos" + +pass=0 +fail=0 +skip=0 +failures=() + +for repo_dir in "$REPOS_DIR"/*/; do + slug="$(basename "$repo_dir")" + + if [[ ! -d "$repo_dir/.harmont" ]]; then + skip=$((skip + 1)) + continue + fi + + echo "══════════════════════════════════════" + echo " $slug" + echo "══════════════════════════════════════" + + if "$SCRIPT_DIR/execute.sh" "$slug"; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + failures+=("$slug") + fi + echo "" +done + +echo "" +echo "── Execution Results ──" +echo "pass: $pass fail: $fail skip: $skip" + +if [[ ${#failures[@]} -gt 0 ]]; then + echo "" + echo "Failures:" + for f in "${failures[@]}"; do + echo " - $f" + done + exit 1 +fi +``` + +**Step 3: Make executable** + +Run: +```bash +chmod +x proving-grounds/scripts/execute.sh proving-grounds/scripts/execute-all.sh +``` + +**Step 4: Commit** + +```bash +git add proving-grounds/scripts/execute.sh proving-grounds/scripts/execute-all.sh +git commit -m "feat: execution-tier test scripts for proving-grounds" +``` + +--- + +### Task 5: Write Rust integration test for compile tier + +**Files:** +- Create: `proving-grounds/tests/compile_fixtures.rs` +- Modify: `cli/Cargo.toml` (add proving-grounds test target if needed) + +**Step 1: Write the Rust compile test** + +This test auto-discovers all repos with `.harmont/pipeline.py` and validates their IR. + +```rust +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +fn proving_grounds_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../proving-grounds/repos") +} + +fn hm_bin() -> String { + std::env::var("HM_BIN").unwrap_or_else(|_| "hm".to_string()) +} + +fn repo_slugs() -> Vec { + let dir = proving_grounds_dir(); + if !dir.exists() { + return vec![]; + } + let mut slugs = vec![]; + for entry in fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let harmont = entry.path().join(".harmont/pipeline.py"); + if harmont.exists() { + slugs.push(entry.file_name().to_string_lossy().to_string()); + } + } + slugs.sort(); + slugs +} + +#[test] +fn all_proving_ground_fixtures_render() { + let slugs = repo_slugs(); + if slugs.is_empty() { + eprintln!("SKIP: no proving-ground fixtures with .harmont/ found"); + return; + } + + let mut failures = vec![]; + + for slug in &slugs { + let dir = proving_grounds_dir().join(slug); + let output = Command::new(hm_bin()) + .args(["render", "ci", "--dir"]) + .arg(&dir) + .output() + .unwrap_or_else(|e| panic!("spawn hm: {e}")); + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + failures.push(format!("{slug}: {stderr}")); + continue; + } + + // Validate JSON parses + let stdout = String::from_utf8_lossy(&output.stdout); + let _: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("{slug}: invalid JSON: {e}")); + } + + if !failures.is_empty() { + panic!( + "{} fixture(s) failed to render:\n{}", + failures.len(), + failures.join("\n") + ); + } +} + +#[test] +fn all_proving_ground_fixtures_have_valid_structure() { + let slugs = repo_slugs(); + if slugs.is_empty() { + return; + } + + for slug in &slugs { + let dir = proving_grounds_dir().join(slug); + let output = Command::new(hm_bin()) + .args(["render", "ci", "--dir"]) + .arg(&dir) + .output() + .unwrap(); + + if !output.status.success() { + continue; // caught by render test above + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let ir: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + + // Validate v0 IR structure + assert_eq!(ir["version"], "0", "{slug}: version != 0"); + + let nodes = ir["graph"]["nodes"].as_array() + .unwrap_or_else(|| panic!("{slug}: missing graph.nodes")); + assert!(!nodes.is_empty(), "{slug}: no nodes"); + + for node in nodes { + let key = node["step"]["key"].as_str() + .unwrap_or_else(|| panic!("{slug}: node missing step.key")); + assert!(!key.is_empty(), "{slug}: empty key"); + + let cmd = node["step"]["cmd"].as_str() + .unwrap_or_else(|| panic!("{slug}: node missing step.cmd")); + assert!(!cmd.is_empty(), "{slug}: empty cmd for key {key}"); + } + + let edges = ir["graph"]["edges"].as_array() + .unwrap_or_else(|| panic!("{slug}: missing graph.edges")); + assert!(!edges.is_empty(), "{slug}: no edges"); + + // No self-loops + for edge in edges { + let src = edge[0].as_u64().unwrap(); + let dst = edge[1].as_u64().unwrap(); + assert_ne!(src, dst, "{slug}: self-loop on node {src}"); + } + } +} +``` + +**Step 2: Run test to verify it compiles (will skip if no fixtures yet)** + +Run: `cargo test -p hm --test compile_fixtures -- --nocapture` +Expected: SKIP message (no fixtures yet) + +**Step 3: Commit** + +```bash +git add proving-grounds/tests/compile_fixtures.rs +git commit -m "feat: compile-tier Rust integration test for proving-grounds" +``` + +--- + +### Task 6: Write pipeline rewrite — BurntSushi/ripgrep (Rust, simple) + +**Files:** +- Create: `proving-grounds/repos/BurntSushi--ripgrep/.harmont/pipeline.py` + +**Step 1: Study the original CI** + +Run: `python proving-grounds/scripts/fetch-workflows.py --slug BurntSushi--ripgrep` +Then read the downloaded workflow YAML to understand what the CI does. + +**Step 2: Write the harmont rewrite** + +```python +"""ripgrep — Rust CLI for recursive regex search.""" +from __future__ import annotations + +import harmont as hm +from harmont.rust import RustToolchain + + +@hm.target() +def project() -> RustToolchain: + return hm.rust.toolchain(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.clippy(), + project.fmt(), + ) +``` + +**Step 3: Verify compile** + +Run: `hm render ci --dir proving-grounds/repos/BurntSushi--ripgrep` +Expected: Valid JSON IR output + +**Step 4: Commit** + +```bash +git add proving-grounds/repos/BurntSushi--ripgrep/ +git commit -m "feat(proving-grounds): add ripgrep pipeline rewrite" +``` + +--- + +### Task 7: Write pipeline rewrite — pallets/flask (Python, simple) + +**Files:** +- Create: `proving-grounds/repos/pallets--flask/.harmont/pipeline.py` + +**Step 1: Fetch and study original CI** + +Run: `python proving-grounds/scripts/fetch-workflows.py --slug pallets--flask` + +**Step 2: Write the harmont rewrite** + +```python +"""Flask — lightweight Python web framework.""" +from __future__ import annotations + +import harmont as hm +from harmont.python import PythonToolchain + + +@hm.target() +def project() -> PythonToolchain: + return hm.python(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: + return ( + project.test(), + project.lint(), + project.typecheck(), + ) +``` + +**Step 3: Verify compile** + +Run: `hm render ci --dir proving-grounds/repos/pallets--flask` + +**Step 4: Commit** + +```bash +git add proving-grounds/repos/pallets--flask/ +git commit -m "feat(proving-grounds): add flask pipeline rewrite" +``` + +--- + +### Task 8: Write pipeline rewrite — expressjs/express (JS, simple) + +**Files:** +- Create: `proving-grounds/repos/expressjs--express/.harmont/pipeline.py` + +**Step 1: Fetch and study original CI** + +Run: `python proving-grounds/scripts/fetch-workflows.py --slug expressjs--express` + +**Step 2: Write the harmont rewrite** + +```python +"""Express — minimal Node.js web framework.""" +from __future__ import annotations + +import harmont as hm +from harmont.npm import NpmProject + + +@hm.target() +def project() -> NpmProject: + return hm.npm(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: + return ( + project.test(), + project.lint(), + ) +``` + +**Step 3: Verify compile, commit** + +--- + +### Task 9: Write pipeline rewrite — gin-gonic/gin (Go, simple) + +**Files:** +- Create: `proving-grounds/repos/gin-gonic--gin/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Gin — HTTP web framework for Go.""" +from __future__ import annotations + +import harmont as hm +from harmont.go import GoToolchain + + +@hm.target() +def project() -> GoToolchain: + return hm.go(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[GoToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.vet(), + project.fmt(), + ) +``` + +**Step 3: Verify compile, commit** + +--- + +### Task 10: Write pipeline rewrite — clap-rs/clap (Rust, simple) + +**Files:** +- Create: `proving-grounds/repos/clap-rs--clap/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""clap — CLI argument parser for Rust.""" +from __future__ import annotations + +import harmont as hm +from harmont.rust import RustToolchain + + +@hm.target() +def project() -> RustToolchain: + return hm.rust.toolchain(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.clippy(), + project.fmt(), + ) +``` + +--- + +### Task 11: Write pipeline rewrite — tokio-rs/tokio (Rust, medium) + +**Files:** +- Create: `proving-grounds/repos/tokio-rs--tokio/.harmont/pipeline.py` + +Tokio is a Cargo workspace with multiple crates. The pipeline needs to build/test the workspace, plus run clippy and fmt. Harmont's Rust toolchain operates at workspace level. + +**Step 2: Write the harmont rewrite** + +```python +"""Tokio — async runtime for Rust.""" +from __future__ import annotations + +import harmont as hm +from harmont.rust import RustToolchain + + +@hm.target() +def project() -> RustToolchain: + return hm.rust.toolchain(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true", "RUSTFLAGS": "-Dwarnings"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.clippy(), + project.fmt(), + ) +``` + +--- + +### Task 12: Write pipeline rewrite — vitejs/vite (TypeScript, medium) + +**Files:** +- Create: `proving-grounds/repos/vitejs--vite/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Vite — next-generation frontend build tool.""" +from __future__ import annotations + +import harmont as hm +from harmont.npm import NpmProject + + +@hm.target() +def project() -> NpmProject: + return hm.npm(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: + return ( + project.run("build"), + project.run("test"), + project.run("lint"), + ) +``` + +--- + +### Task 13: Write pipeline rewrite — hashicorp/terraform (Go, medium) + +**Files:** +- Create: `proving-grounds/repos/hashicorp--terraform/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Terraform — infrastructure as code.""" +from __future__ import annotations + +import harmont as hm +from harmont.go import GoToolchain + + +@hm.target() +def project() -> GoToolchain: + return hm.go(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[GoToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.vet(), + project.fmt(), + ) +``` + +--- + +### Task 14: Write pipeline rewrite — django/django (Python, medium) + +**Files:** +- Create: `proving-grounds/repos/django--django/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Django — Python web framework.""" +from __future__ import annotations + +import harmont as hm +from harmont.python import PythonToolchain + + +@hm.target() +def project() -> PythonToolchain: + return hm.python(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: + return ( + project.test(), + project.lint(), + project.fmt(), + ) +``` + +--- + +### Task 15: Write pipeline rewrite — astral-sh/ruff (Rust, medium) + +**Files:** +- Create: `proving-grounds/repos/astral-sh--ruff/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Ruff — fast Python linter/formatter written in Rust.""" +from __future__ import annotations + +import harmont as hm +from harmont.rust import RustToolchain + + +@hm.target() +def project() -> RustToolchain: + return hm.rust.toolchain(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.clippy(), + project.fmt(), + ) +``` + +--- + +### Task 16: Write pipeline rewrite — jekyll/jekyll (Ruby, simple) + +**Files:** +- Create: `proving-grounds/repos/jekyll--jekyll/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Jekyll — static site generator in Ruby.""" +from __future__ import annotations + +import harmont as hm +from harmont.ruby import RubyToolchain + + +@hm.target() +def project() -> RubyToolchain: + return hm.ruby(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[RubyToolchain]) -> tuple[hm.Step, ...]: + return ( + project.test(), + project.lint(), + ) +``` + +--- + +### Task 17: Write pipeline rewrite — starship/starship (Rust, medium) + +**Files:** +- Create: `proving-grounds/repos/starship--starship/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Starship — cross-shell prompt.""" +from __future__ import annotations + +import harmont as hm +from harmont.rust import RustToolchain + + +@hm.target() +def project() -> RustToolchain: + return hm.rust.toolchain(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + return ( + project.build(), + project.test(), + project.clippy(), + project.fmt(), + ) +``` + +--- + +### Task 18: Write pipeline rewrite — grafana/grafana (Go+TS, complex monorepo) + +**Files:** +- Create: `proving-grounds/repos/grafana--grafana/.harmont/pipeline.py` + +Grafana is a Go backend + TypeScript frontend monorepo. This demonstrates harmont's multi-toolchain composition — the same pattern as the `zig-js` example. + +**Step 2: Write the harmont rewrite** + +```python +"""Grafana — observability platform (Go + TypeScript monorepo).""" +from __future__ import annotations + +from typing import Annotated + +import harmont as hm +from harmont.go import GoToolchain +from harmont.npm import NpmProject + + +@hm.target() +def backend() -> GoToolchain: + return hm.go(path=".") + + +@hm.target() +def frontend() -> NpmProject: + return hm.npm(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci( + backend: hm.Target[GoToolchain], + frontend: hm.Target[NpmProject], +) -> tuple[hm.Step, ...]: + return ( + # Go backend + backend.build(), + backend.test(), + backend.vet(), + # TypeScript frontend + frontend.run("build"), + frontend.run("test"), + frontend.run("lint"), + ) +``` + +--- + +### Task 19: Write pipeline rewrite — fastapi/fastapi (Python, simple) + +**Files:** +- Create: `proving-grounds/repos/fastapi--fastapi/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""FastAPI — modern Python web framework.""" +from __future__ import annotations + +import harmont as hm +from harmont.python import PythonToolchain + + +@hm.target() +def project() -> PythonToolchain: + return hm.python(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="master")], +) +def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: + return ( + project.test(), + project.lint(), + project.fmt(), + project.typecheck(), + ) +``` + +--- + +### Task 20: Write pipeline rewrite — sveltejs/svelte (TypeScript, medium) + +**Files:** +- Create: `proving-grounds/repos/sveltejs--svelte/.harmont/pipeline.py` + +**Step 2: Write the harmont rewrite** + +```python +"""Svelte — cybernetically enhanced web apps.""" +from __future__ import annotations + +import harmont as hm +from harmont.npm import NpmProject + + +@hm.target() +def project() -> NpmProject: + return hm.npm(path=".") + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", + triggers=[hm.push(branch="main")], +) +def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: + return ( + project.run("build"), + project.run("test"), + project.run("lint"), + ) +``` + +--- + +### Task 21: Batch commit all pipeline rewrites + +**Step 1: Verify all fixtures compile** + +Run: `proving-grounds/scripts/compile-all.sh` +Expected: All 15 PASS + +**Step 2: Commit all rewrites** + +```bash +git add proving-grounds/repos/ +git commit -m "feat(proving-grounds): initial 15 pipeline rewrites + +Repos: ripgrep, flask, express, gin, clap, tokio, vite, terraform, +django, ruff, jekyll, starship, grafana, fastapi, svelte" +``` + +--- + +### Task 22: Add CI integration for compile tier + +**Files:** +- Modify: `.github/workflows/ci.yml` (or create proving-grounds-specific workflow) + +**Step 1: Add proving-grounds compile step to CI** + +Add a job to the CI workflow: + +```yaml + proving-grounds-compile: + name: Proving Grounds (compile) + runs-on: ubuntu-latest + needs: [build] # depends on hm binary being built + steps: + - uses: actions/checkout@v4 + - name: Download hm binary + uses: actions/download-artifact@v4 + with: + name: hm-linux + path: /usr/local/bin/ + - run: chmod +x /usr/local/bin/hm + - name: Install Python deps + run: | + pip install harmont + - name: Compile all proving-ground fixtures + run: proving-grounds/scripts/compile-all.sh +``` + +**Step 2: Commit** + +```bash +git add .github/workflows/ +git commit -m "ci: add proving-grounds compile tier to CI" +``` + +--- + +### Task 23: Add nightly execution tier workflow + +**Files:** +- Create: `.github/workflows/proving-grounds-nightly.yml` + +**Step 1: Write the nightly workflow** + +```yaml +name: Proving Grounds (nightly) + +on: + schedule: + - cron: '0 4 * * *' # 4am UTC daily + workflow_dispatch: + +jobs: + execute: + name: Execute proving-ground pipelines + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - name: Build hm + run: cargo build --release -p hm + - name: Install Python deps + run: pip install harmont + - name: Run execution tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HM_BIN: ./target/release/hm + run: proving-grounds/scripts/execute-all.sh +``` + +**Step 2: Commit** + +```bash +git add .github/workflows/proving-grounds-nightly.yml +git commit -m "ci: add nightly proving-grounds execution workflow" +``` + +--- + +## Expansion Guide: Adding More Repos + +For each new repo (remaining 85 of 100): + +1. **Verify it's in `manifest.toml`** — entry exists with correct metadata +2. **Fetch workflows**: `python proving-grounds/scripts/fetch-workflows.py --slug ` +3. **Study the original CI** — read the downloaded YAML in `repos//workflows/` +4. **Write `.harmont/pipeline.py`** — follow the pattern from existing rewrites: + - Pick the right toolchain(s) for the repo's language + - Express the core pipeline: build → test → lint → fmt + - For monorepos, use multi-toolchain composition (Task 18 pattern) + - For repos with custom build steps, use `sh()` directly +5. **Verify compile**: `hm render ci --dir proving-grounds/repos/` +6. **Update manifest status**: Change `status = "pending"` to `"compile-only"` or `"passing"` +7. **Commit** + +### When a repo's build needs custom steps + +Some repos need steps beyond what toolchains provide. Use `sh()`: + +```python +@hm.pipeline("ci", default_image="ubuntu:24.04") +def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: + installed = project.installed + custom = installed.sh("cargo xtask codegen", label=":codegen:") + return ( + custom, + project.build(), + project.test(), + ) +``` + +### What harmont currently CANNOT express + +These features appear in real-world CI but have no harmont equivalent yet. When rewriting, express only what harmont supports: + +| GHA Feature | Harmont Status | Workaround | +|-------------|---------------|------------| +| Matrix builds | Not supported | Write single-config pipeline (one OS, one version) | +| Conditional steps (`if:`) | Not supported | Include unconditionally or omit | +| Artifact upload/download | Not needed | Harmont uses snapshot lineage | +| Reusable workflows | Not supported | Inline the logic | +| Self-hosted runners | Not supported | Use default `ubuntu:24.04` image | +| Service containers | Not supported | Install services in pipeline steps | + +### Priority order for remaining 85 repos + +1. **Simple repos with matching toolchains** — bat, fd, zoxide, serde, requests, black, poetry, got, chalk, prettier, chi, testify, fzf (quick wins, ~5 min each) +2. **Medium repos with matching toolchains** — nushell, helix, ruff, pnpm, deno, prometheus, vault, etcd (slightly more complex but same pattern) +3. **Complex repos** — these may surface DSL gaps; each needs careful study of original CI +4. **Extreme repos** — mostly reference/aspiration; many use non-GHA CI systems + +--- + +## Success Metrics + +| Metric | Target | +|--------|--------| +| Repos with compile-only status | 80/100 within 2 weeks | +| Repos with passing execution | 30/100 within 1 month | +| Compile tier CI green | Every commit | +| Execution tier nightly green | 80% of repos with pipelines | +| DSL gaps identified | Documented per repo in metadata.toml | diff --git a/proving-grounds/.harmont/pipeline.py b/proving-grounds/.harmont/pipeline.py new file mode 100644 index 00000000..f4ac3ace --- /dev/null +++ b/proving-grounds/.harmont/pipeline.py @@ -0,0 +1,128 @@ +"""Proving grounds — exercise harmont's DSL against real-world OSS repos. + +Each repo is cloned into /opt/ and built/tested/linted using +harmont's toolchain abstractions. All repos run as parallel chains +from a shared apt base. +""" +from __future__ import annotations + +from datetime import timedelta +from typing import Annotated + +import harmont as hm + +RUST_REPOS = [ + ("ripgrep", "https://github.com/BurntSushi/ripgrep"), + ("clap", "https://github.com/clap-rs/clap"), + ("tokio", "https://github.com/tokio-rs/tokio"), + ("starship", "https://github.com/starship/starship"), + ("ruff", "https://github.com/astral-sh/ruff"), +] + +PYTHON_REPOS = [ + ("flask", "https://github.com/pallets/flask"), + ("django", "https://github.com/django/django"), + ("fastapi", "https://github.com/fastapi/fastapi"), +] + +NPM_REPOS = [ + ("express", "https://github.com/expressjs/express"), + ("vite", "https://github.com/vitejs/vite"), + ("svelte", "https://github.com/sveltejs/svelte"), +] + +GO_REPOS = [ + ("gin", "https://github.com/gin-gonic/gin"), + ("terraform", "https://github.com/hashicorp/terraform"), +] + +RUBY_REPOS = [ + ("jekyll", "https://github.com/jekyll/jekyll"), +] + + +@hm.target() +def apt(root: Annotated[hm.Step, hm.BaseImage("ubuntu:24.04")]) -> hm.Step: + return root.sh( + "apt-get update && " + "apt-get install -y --no-install-recommends " + "git ca-certificates curl build-essential pkg-config", + label=":apt: base", + cache=hm.ttl(timedelta(days=1)), + ) + + +def _clone(base: hm.Step, name: str, url: str) -> hm.Step: + return base.fork(name).sh( + f"git clone --depth 1 {url} /opt/{name}", + label=f":git: {name}", + ) + + +def _rust_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: + cloned = _clone(apt, name, url) + tc = hm.rust.toolchain(path=f"/opt/{name}", base=cloned) + return [tc.build(), tc.test(), tc.clippy(), tc.fmt()] + + +def _python_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: + cloned = _clone(apt, name, url) + tc = hm.python(path=f"/opt/{name}", base=cloned) + return [tc.test(), tc.lint(), tc.typecheck()] + + +def _npm_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: + cloned = _clone(apt, name, url) + project = hm.npm(path=f"/opt/{name}", base=cloned) + return [project.run("build"), project.run("test"), project.run("lint")] + + +def _go_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: + cloned = _clone(apt, name, url) + tc = hm.go(path=f"/opt/{name}", base=cloned) + return [tc.build(), tc.test(), tc.vet(), tc.fmt()] + + +def _ruby_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: + cloned = _clone(apt, name, url) + project = hm.ruby(path=f"/opt/{name}", base=cloned) + return [project.test(), project.lint()] + + +@hm.pipeline( + "ci", + env={"CI": "true"}, + default_image="ubuntu:24.04", +) +def ci(apt: hm.Target[hm.Step]) -> tuple[hm.Step, ...]: + leaves: list[hm.Step] = [] + + for name, url in RUST_REPOS: + leaves.extend(_rust_leaves(apt, name, url)) + + for name, url in PYTHON_REPOS: + leaves.extend(_python_leaves(apt, name, url)) + + for name, url in NPM_REPOS: + leaves.extend(_npm_leaves(apt, name, url)) + + for name, url in GO_REPOS: + leaves.extend(_go_leaves(apt, name, url)) + + for name, url in RUBY_REPOS: + leaves.extend(_ruby_leaves(apt, name, url)) + + # Grafana — multi-toolchain monorepo (Go + npm) + grafana_clone = _clone(apt, "grafana", "https://github.com/grafana/grafana") + grafana_go = hm.go(path="/opt/grafana", base=grafana_clone) + grafana_npm = hm.npm(path="/opt/grafana", base=grafana_clone) + leaves.extend([ + grafana_go.build(), + grafana_go.test(), + grafana_go.vet(), + grafana_npm.run("build"), + grafana_npm.run("test"), + grafana_npm.run("lint"), + ]) + + return tuple(leaves) From e8c6493d8e7d949b7ba76d53df943aefc7fd7e9c Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 6 Jun 2026 15:12:42 -0700 Subject: [PATCH 2/3] fix: audit all proving-ground repos, pin SHAs, use correct build commands - Drop 4 repos with wrong package managers (django=pip, vite/svelte=pnpm, grafana=yarn4). Down to 11 repos that match harmont's toolchains. - Pin every clone to a specific commit SHA (no more HEAD drift). - Per-repo audited commands from actual CI configs: - ripgrep: skip clippy (not in their CI), workspace build/test/fmt - clap: feature flags, -A deprecated for clippy - tokio: --features full,test-util, rustfmt directly (not cargo fmt) - starship: --locked, libdbus-1-dev in apt base - ruff: pinned to Rust 1.96, --all-features --locked - flask: standard hm.python() works - fastapi: custom pytest flags + PYTHONPATH, ty check on package dir - express: no build script, just test + lint - gin: standard hm.go() works - terraform: multi-module test loop over go list -m - jekyll: TZ=UTC for tests, rubocop for lint - Delete stale plan doc (described architecture that doesn't exist) --- docs/plans/2026-06-04-proving-grounds.md | 2234 ---------------------- proving-grounds/.harmont/pipeline.py | 282 ++- 2 files changed, 199 insertions(+), 2317 deletions(-) delete mode 100644 docs/plans/2026-06-04-proving-grounds.md diff --git a/docs/plans/2026-06-04-proving-grounds.md b/docs/plans/2026-06-04-proving-grounds.md deleted file mode 100644 index e16e8408..00000000 --- a/docs/plans/2026-06-04-proving-grounds.md +++ /dev/null @@ -1,2234 +0,0 @@ -# Proving Grounds Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Build a test suite of 100 real-world OSS repos whose CI pipelines are rewritten in harmont's DSL, proving harmont can express and execute real-world build pipelines. - -**Architecture:** Two-tier testing. *Compile tier* (fast, no Docker): render each pipeline to IR via `hm render`, validate structure via insta snapshots + structural assertions. *Execution tier* (slow, Docker): clone repo at pinned SHA, run `hm run`, verify exit 0. All fixtures live in `proving-grounds/` at repo root — pipeline.py rewrites checked in, original GHA YAML stored for reference, manifest.toml tracks metadata. Initial batch of 15 repos hand-written; remaining 85 added incrementally. - -**Tech Stack:** Python DSL (harmont package), Rust integration tests (insta), shell scripts (fetch/execute), TOML manifests. - ---- - -## Directory Layout - -``` -proving-grounds/ - manifest.toml # All 100 repos: slug, url, sha, lang, complexity, status - scripts/ - fetch-workflows.py # Download GHA YAML from GitHub API - compile-all.sh # hm render every fixture, report pass/fail - execute.sh # Clone + hm run for a single repo - execute-all.sh # Batch execution (nightly) - repos/ - --/ - metadata.toml # Per-repo: url, sha, language, complexity, notes - workflows/ # Original GHA YAML (reference only) - ci.yml - .harmont/ - pipeline.py # Harmont DSL rewrite - tests/ - compile_fixtures.rs # Rust integration test: render + validate IR - snapshots/ # insta snapshot files -``` - ---- - -### Task 1: Create directory structure and manifest format - -**Files:** -- Create: `proving-grounds/manifest.toml` -- Create: `proving-grounds/repos/.gitkeep` (placeholder) - -**Step 1: Create the directory tree** - -Run: -```bash -mkdir -p proving-grounds/{scripts,repos,tests/snapshots} -``` - -**Step 2: Write the manifest with all 100 repos** - -Create `proving-grounds/manifest.toml`: - -```toml -# Proving Grounds Manifest -# status: "pending" | "compile-only" | "passing" | "failing" | "skip" -# complexity: "simple" | "medium" | "complex" | "extreme" - -# ── SIMPLE (30) ───────────────────────────────────────────────────── -[[repos]] -slug = "BurntSushi--ripgrep" -url = "https://github.com/BurntSushi/ripgrep" -sha = "" # pin after first fetch -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "clippy"] - -[[repos]] -slug = "sharkdp--bat" -url = "https://github.com/sharkdp/bat" -sha = "" -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "clippy"] - -[[repos]] -slug = "sharkdp--fd" -url = "https://github.com/sharkdp/fd" -sha = "" -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "ajeetdsouza--zoxide" -url = "https://github.com/ajeetdsouza/zoxide" -sha = "" -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "clap-rs--clap" -url = "https://github.com/clap-rs/clap" -sha = "" -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "clippy"] - -[[repos]] -slug = "serde-rs--serde" -url = "https://github.com/serde-rs/serde" -sha = "" -language = "rust" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "msrv"] - -[[repos]] -slug = "pallets--flask" -url = "https://github.com/pallets/flask" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "psf--requests" -url = "https://github.com/psf/requests" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "psf--black" -url = "https://github.com/psf/black" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "fuzz"] - -[[repos]] -slug = "python-poetry--poetry" -url = "https://github.com/python-poetry/poetry" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "fastapi--fastapi" -url = "https://github.com/fastapi/fastapi" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test", "typecheck"] - -[[repos]] -slug = "tiangolo--sqlmodel" -url = "https://github.com/tiangolo/sqlmodel" -sha = "" -language = "python" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "expressjs--express" -url = "https://github.com/expressjs/express" -sha = "" -language = "javascript" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "sindresorhus--got" -url = "https://github.com/sindresorhus/got" -sha = "" -language = "typescript" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "chalk--chalk" -url = "https://github.com/chalk/chalk" -sha = "" -language = "javascript" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "prettier--prettier" -url = "https://github.com/prettier/prettier" -sha = "" -language = "javascript" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "gin-gonic--gin" -url = "https://github.com/gin-gonic/gin" -sha = "" -language = "go" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "go-chi--chi" -url = "https://github.com/go-chi/chi" -sha = "" -language = "go" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "stretchr--testify" -url = "https://github.com/stretchr/testify" -sha = "" -language = "go" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "junegunn--fzf" -url = "https://github.com/junegunn/fzf" -sha = "" -language = "go" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "jqlang--jq" -url = "https://github.com/jqlang/jq" -sha = "" -language = "c" -complexity = "simple" -status = "pending" -ci_features = ["build", "test"] - -[[repos]] -slug = "jekyll--jekyll" -url = "https://github.com/jekyll/jekyll" -sha = "" -language = "ruby" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "rails--rails" -url = "https://github.com/rails/rails" -sha = "" -language = "ruby" -complexity = "simple" -status = "pending" -ci_features = ["test"] - -[[repos]] -slug = "google--gson" -url = "https://github.com/google/gson" -sha = "" -language = "java" -complexity = "simple" -status = "pending" -ci_features = ["test"] - -[[repos]] -slug = "square--okhttp" -url = "https://github.com/square/okhttp" -sha = "" -language = "kotlin" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "google--guava" -url = "https://github.com/google/guava" -sha = "" -language = "java" -complexity = "simple" -status = "pending" -ci_features = ["test"] - -[[repos]] -slug = "lodash--lodash" -url = "https://github.com/lodash/lodash" -sha = "" -language = "javascript" -complexity = "simple" -status = "pending" -ci_features = ["lint", "test"] - -[[repos]] -slug = "redis--redis" -url = "https://github.com/redis/redis" -sha = "" -language = "c" -complexity = "simple" -status = "pending" -ci_features = ["build", "test"] - -[[repos]] -slug = "JetBrains--kotlin" -url = "https://github.com/JetBrains/kotlin" -sha = "" -language = "kotlin" -complexity = "simple" -status = "pending" -ci_features = ["build", "test"] - -[[repos]] -slug = "Alamofire--Alamofire" -url = "https://github.com/Alamofire/Alamofire" -sha = "" -language = "swift" -complexity = "simple" -status = "pending" -ci_features = ["build", "test"] - -# ── MEDIUM (30) ───────────────────────────────────────────────────── -[[repos]] -slug = "tokio-rs--tokio" -url = "https://github.com/tokio-rs/tokio" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "multi-crate"] - -[[repos]] -slug = "starship--starship" -url = "https://github.com/starship/starship" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching", "release"] - -[[repos]] -slug = "nushell--nushell" -url = "https://github.com/nushell/nushell" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching", "multi-crate"] - -[[repos]] -slug = "helix-editor--helix" -url = "https://github.com/helix-editor/helix" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "clippy", "caching"] - -[[repos]] -slug = "astral-sh--ruff" -url = "https://github.com/astral-sh/ruff" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching", "benchmark"] - -[[repos]] -slug = "django--django" -url = "https://github.com/django/django" -sha = "" -language = "python" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "db-backends"] - -[[repos]] -slug = "scikit-learn--scikit-learn" -url = "https://github.com/scikit-learn/scikit-learn" -sha = "" -language = "python" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "doc-build"] - -[[repos]] -slug = "pandas-dev--pandas" -url = "https://github.com/pandas-dev/pandas" -sha = "" -language = "python" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "benchmark"] - -[[repos]] -slug = "vercel--next.js" -url = "https://github.com/vercel/next.js" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "turbopack"] - -[[repos]] -slug = "sveltejs--svelte" -url = "https://github.com/sveltejs/svelte" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "multi-package"] - -[[repos]] -slug = "vitejs--vite" -url = "https://github.com/vitejs/vite" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "ecosystem-ci"] - -[[repos]] -slug = "withastro--astro" -url = "https://github.com/withastro/astro" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "multi-package"] - -[[repos]] -slug = "pnpm--pnpm" -url = "https://github.com/pnpm/pnpm" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching"] - -[[repos]] -slug = "denoland--deno" -url = "https://github.com/denoland/deno" -sha = "" -language = "rust" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching", "multi-suite"] - -[[repos]] -slug = "prometheus--prometheus" -url = "https://github.com/prometheus/prometheus" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "docker-build"] - -[[repos]] -slug = "hashicorp--terraform" -url = "https://github.com/hashicorp/terraform" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os", "caching", "acceptance"] - -[[repos]] -slug = "hashicorp--vault" -url = "https://github.com/hashicorp/vault" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "integration"] - -[[repos]] -slug = "etcd-io--etcd" -url = "https://github.com/etcd-io/etcd" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "integration"] - -[[repos]] -slug = "golang--go" -url = "https://github.com/golang/go" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["multi-builder", "caching"] - -[[repos]] -slug = "curl--curl" -url = "https://github.com/curl/curl" -sha = "" -language = "c" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os-compiler", "caching"] - -[[repos]] -slug = "facebook--zstd" -url = "https://github.com/facebook/zstd" -sha = "" -language = "c" -complexity = "medium" -status = "pending" -ci_features = ["matrix-os-compiler", "caching"] - -[[repos]] -slug = "spring-projects--spring-boot" -url = "https://github.com/spring-projects/spring-boot" -sha = "" -language = "java" -complexity = "medium" -status = "pending" -ci_features = ["matrix-jdk", "caching", "integration"] - -[[repos]] -slug = "elastic--elasticsearch" -url = "https://github.com/elastic/elasticsearch" -sha = "" -language = "java" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "integration"] - -[[repos]] -slug = "apache--kafka" -url = "https://github.com/apache/kafka" -sha = "" -language = "java" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "system-tests"] - -[[repos]] -slug = "vapor--vapor" -url = "https://github.com/vapor/vapor" -sha = "" -language = "swift" -complexity = "medium" -status = "pending" -ci_features = ["matrix-swift", "linux-macos"] - -[[repos]] -slug = "storybookjs--storybook" -url = "https://github.com/storybookjs/storybook" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "multi-framework"] - -[[repos]] -slug = "angular--angular" -url = "https://github.com/angular/angular" -sha = "" -language = "typescript" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "bazel"] - -[[repos]] -slug = "pytorch--pytorch" -url = "https://github.com/pytorch/pytorch" -sha = "" -language = "python" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "caching", "cuda", "artifacts"] - -[[repos]] -slug = "goreleaser--goreleaser" -url = "https://github.com/goreleaser/goreleaser" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["matrix", "docker-multi-arch", "release"] - -[[repos]] -slug = "docker--compose" -url = "https://github.com/docker/compose" -sha = "" -language = "go" -complexity = "medium" -status = "pending" -ci_features = ["multi-platform", "docker-in-docker", "release"] - -# ── COMPLEX (25) ──────────────────────────────────────────────────── -[[repos]] -slug = "rust-lang--rust" -url = "https://github.com/rust-lang/rust" -sha = "" -language = "rust" -complexity = "complex" -status = "pending" -ci_features = ["massive-matrix", "cross-compilation", "artifacts", "bors"] - -[[repos]] -slug = "alacritty--alacritty" -url = "https://github.com/alacritty/alacritty" -sha = "" -language = "rust" -complexity = "complex" -status = "pending" -ci_features = ["cross-platform", "release-artifacts", "caching"] - -[[repos]] -slug = "tauri-apps--tauri" -url = "https://github.com/tauri-apps/tauri" -sha = "" -language = "rust" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "release-automation", "docker", "cross-compile"] - -[[repos]] -slug = "zed-industries--zed" -url = "https://github.com/zed-industries/zed" -sha = "" -language = "rust" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "release", "caching", "signing"] - -[[repos]] -slug = "oven-sh--bun" -url = "https://github.com/oven-sh/bun" -sha = "" -language = "zig" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "cross-compilation", "docker", "release"] - -[[repos]] -slug = "huggingface--transformers" -url = "https://github.com/huggingface/transformers" -sha = "" -language = "python" -complexity = "complex" -status = "pending" -ci_features = ["matrix", "gpu-runners", "doc-deploy", "model-tests"] - -[[repos]] -slug = "tensorflow--tensorflow" -url = "https://github.com/tensorflow/tensorflow" -sha = "" -language = "cpp" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "gpu", "docker", "release"] - -[[repos]] -slug = "python--cpython" -url = "https://github.com/python/cpython" -sha = "" -language = "c" -complexity = "complex" -status = "pending" -ci_features = ["matrix-os-compiler", "cross-compilation", "release", "extensive-tests"] - -[[repos]] -slug = "facebook--react" -url = "https://github.com/facebook/react" -sha = "" -language = "javascript" -complexity = "complex" -status = "pending" -ci_features = ["multi-package", "artifacts", "deploy", "canary"] - -[[repos]] -slug = "microsoft--TypeScript" -url = "https://github.com/microsoft/TypeScript" -sha = "" -language = "typescript" -complexity = "complex" -status = "pending" -ci_features = ["matrix", "release", "benchmarks", "artifacts"] - -[[repos]] -slug = "eslint--eslint" -url = "https://github.com/eslint/eslint" -sha = "" -language = "javascript" -complexity = "complex" -status = "pending" -ci_features = ["matrix", "caching", "semantic-release"] - -[[repos]] -slug = "prisma--prisma" -url = "https://github.com/prisma/prisma" -sha = "" -language = "typescript" -complexity = "complex" -status = "pending" -ci_features = ["matrix-db", "docker-test-dbs", "release"] - -[[repos]] -slug = "containerd--containerd" -url = "https://github.com/containerd/containerd" -sha = "" -language = "go" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "cross-compilation", "integration", "release"] - -[[repos]] -slug = "grpc--grpc" -url = "https://github.com/grpc/grpc" -sha = "" -language = "cpp" -complexity = "complex" -status = "pending" -ci_features = ["matrix-languages", "multi-platform", "docker"] - -[[repos]] -slug = "llvm--llvm-project" -url = "https://github.com/llvm/llvm-project" -sha = "" -language = "cpp" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "massive-matrix", "sanitizers", "release"] - -[[repos]] -slug = "FFmpeg--FFmpeg" -url = "https://github.com/FFmpeg/FFmpeg" -sha = "" -language = "c" -complexity = "complex" -status = "pending" -ci_features = ["cross-compilation", "multi-platform", "configure-matrix"] - -[[repos]] -slug = "flutter--flutter" -url = "https://github.com/flutter/flutter" -sha = "" -language = "dart" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "device-testing", "release-channels"] - -[[repos]] -slug = "expo--expo" -url = "https://github.com/expo/expo" -sha = "" -language = "typescript" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform-mobile", "release", "monorepo", "caching"] - -[[repos]] -slug = "grafana--grafana" -url = "https://github.com/grafana/grafana" -sha = "" -language = "go" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "docker", "release", "frontend-backend"] - -[[repos]] -slug = "cockroachdb--cockroach" -url = "https://github.com/cockroachdb/cockroach" -sha = "" -language = "go" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "docker", "release", "integration"] - -[[repos]] -slug = "tikv--tikv" -url = "https://github.com/tikv/tikv" -sha = "" -language = "rust" -complexity = "complex" -status = "pending" -ci_features = ["matrix", "cross-compilation", "integration", "release"] - -[[repos]] -slug = "apache--spark" -url = "https://github.com/apache/spark" -sha = "" -language = "scala" -complexity = "complex" -status = "pending" -ci_features = ["matrix-jdk-scala-hadoop", "docker", "extensive-tests"] - -[[repos]] -slug = "SignalApp--Signal-Android" -url = "https://github.com/SignalApp/Signal-Android" -sha = "" -language = "kotlin" -complexity = "complex" -status = "pending" -ci_features = ["android-build", "signing", "artifacts", "release"] - -[[repos]] -slug = "neovim--neovim" -url = "https://github.com/neovim/neovim" -sha = "" -language = "c" -complexity = "complex" -status = "pending" -ci_features = ["multi-platform", "caching", "release", "functional-tests"] - -[[repos]] -slug = "supabase--supabase" -url = "https://github.com/supabase/supabase" -sha = "" -language = "typescript" -complexity = "complex" -status = "pending" -ci_features = ["monorepo", "docker", "multi-service", "release"] - -# ── EXTREME (15) ──────────────────────────────────────────────────── -[[repos]] -slug = "kubernetes--kubernetes" -url = "https://github.com/kubernetes/kubernetes" -sha = "" -language = "go" -complexity = "extreme" -status = "pending" -ci_features = ["prow", "massive-matrix", "self-hosted", "conditional", "conformance"] - -[[repos]] -slug = "NixOS--nixpkgs" -url = "https://github.com/NixOS/nixpkgs" -sha = "" -language = "nix" -complexity = "extreme" -status = "pending" -ci_features = ["hydra", "80k-packages", "self-hosted", "conditional"] - -[[repos]] -slug = "Homebrew--homebrew-core" -url = "https://github.com/Homebrew/homebrew-core" -sha = "" -language = "ruby" -complexity = "extreme" -status = "pending" -ci_features = ["massive-formula-matrix", "bottle-builds", "self-hosted"] - -[[repos]] -slug = "vercel--turborepo" -url = "https://github.com/vercel/turborepo" -sha = "" -language = "rust" -complexity = "extreme" -status = "pending" -ci_features = ["monorepo", "reusable-workflows", "remote-caching"] - -[[repos]] -slug = "nrwl--nx" -url = "https://github.com/nrwl/nx" -sha = "" -language = "typescript" -complexity = "extreme" -status = "pending" -ci_features = ["monorepo", "affected-ci", "distributed-execution"] - -[[repos]] -slug = "chromium--chromium" -url = "https://github.com/nicedoc/nicedoc.io" -sha = "" -language = "cpp" -complexity = "extreme" -status = "pending" -ci_features = ["luci", "massive-matrix", "self-hosted", "sanitizers"] -notes = "chromium uses LUCI, not GHA - reference only" - -[[repos]] -slug = "microsoft--vscode" -url = "https://github.com/microsoft/vscode" -sha = "" -language = "typescript" -complexity = "extreme" -status = "pending" -ci_features = ["multi-platform", "electron", "release", "self-hosted", "insider-builds"] - -[[repos]] -slug = "facebook--react-native" -url = "https://github.com/facebook/react-native" -sha = "" -language = "javascript" -complexity = "extreme" -status = "pending" -ci_features = ["multi-platform", "self-hosted", "conditional", "hermes"] - -[[repos]] -slug = "rust-lang--cargo" -url = "https://github.com/rust-lang/cargo" -sha = "" -language = "rust" -complexity = "extreme" -status = "pending" -ci_features = ["reusable-workflows", "matrix", "cross-platform", "bors"] - -[[repos]] -slug = "actions--runner" -url = "https://github.com/actions/runner" -sha = "" -language = "csharp" -complexity = "extreme" -status = "pending" -ci_features = ["self-referential", "multi-platform", "release", "reusable-workflows"] - -[[repos]] -slug = "github--codeql" -url = "https://github.com/github/codeql" -sha = "" -language = "codeql" -complexity = "extreme" -status = "pending" -ci_features = ["reusable-workflows", "multi-language-matrix", "composite-actions"] - -[[repos]] -slug = "nixos--nix" -url = "https://github.com/NixOS/nix" -sha = "" -language = "cpp" -complexity = "extreme" -status = "pending" -ci_features = ["multi-platform", "self-hosted", "flake-ci", "docker", "release"] - -[[repos]] -slug = "envoyproxy--envoy" -url = "https://github.com/envoyproxy/envoy" -sha = "" -language = "cpp" -complexity = "extreme" -status = "pending" -ci_features = ["multi-platform", "bazel", "docker", "self-hosted", "sanitizers"] - -[[repos]] -slug = "pulumi--pulumi" -url = "https://github.com/pulumi/pulumi" -sha = "" -language = "go" -complexity = "extreme" -status = "pending" -ci_features = ["monorepo", "multi-language-sdk", "integration", "release", "reusable-workflows"] - -[[repos]] -slug = "hashicorp--nomad" -url = "https://github.com/hashicorp/nomad" -sha = "" -language = "go" -complexity = "extreme" -status = "pending" -ci_features = ["multi-platform", "integration", "self-hosted", "conditional", "release"] -``` - -**Step 3: Commit** - -```bash -git add proving-grounds/ -git commit -m "feat: proving-grounds scaffold with 100-repo manifest" -``` - ---- - -### Task 2: Write the workflow fetch script - -**Files:** -- Create: `proving-grounds/scripts/fetch-workflows.py` - -**Step 1: Write the fetch script** - -```python -#!/usr/bin/env python3 -"""Fetch .github/workflows/ from repos listed in manifest.toml. - -Usage: - python fetch-workflows.py [--token GITHUB_TOKEN] [--slug SLUG] - -Without --slug, fetches all repos with status != "skip". -With --slug, fetches only that repo. -Pins SHA to HEAD of default branch if sha is empty. -""" -import argparse -import json -import os -import sys -import urllib.request -from pathlib import Path - -MANIFEST = Path(__file__).resolve().parent.parent / "manifest.toml" -REPOS_DIR = Path(__file__).resolve().parent.parent / "repos" - - -def load_manifest(): - # Minimal TOML parser for our simple format — avoids external dep - import tomllib - return tomllib.loads(MANIFEST.read_text()) - - -def gh_api(path: str, token: str | None) -> dict: - url = f"https://api.github.com{path}" - req = urllib.request.Request(url) - req.add_header("Accept", "application/vnd.github.v3+json") - if token: - req.add_header("Authorization", f"token {token}") - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) - - -def gh_raw(owner: str, repo: str, sha: str, path: str, token: str | None) -> bytes: - url = f"https://raw.githubusercontent.com/{owner}/{repo}/{sha}/{path}" - req = urllib.request.Request(url) - if token: - req.add_header("Authorization", f"token {token}") - with urllib.request.urlopen(req) as resp: - return resp.read() - - -def fetch_repo(entry: dict, token: str | None): - slug = entry["slug"] - url = entry["url"] - parts = url.rstrip("/").split("/") - owner, repo = parts[-2], parts[-1] - - sha = entry.get("sha", "") - if not sha: - info = gh_api(f"/repos/{owner}/{repo}", token) - sha = gh_api( - f"/repos/{owner}/{repo}/commits/{info['default_branch']}", token - )["sha"] - print(f" pinned {slug} -> {sha[:12]}") - - repo_dir = REPOS_DIR / slug - wf_dir = repo_dir / "workflows" - wf_dir.mkdir(parents=True, exist_ok=True) - - # Write metadata - meta = repo_dir / "metadata.toml" - meta.write_text( - f'url = "{url}"\n' - f'sha = "{sha}"\n' - f'language = "{entry["language"]}"\n' - f'complexity = "{entry["complexity"]}"\n' - ) - - # Fetch workflow files - try: - contents = gh_api( - f"/repos/{owner}/{repo}/contents/.github/workflows?ref={sha}", token - ) - except Exception as e: - print(f" WARN: no workflows for {slug}: {e}") - return sha - - for item in contents: - if item["name"].endswith((".yml", ".yaml")): - data = gh_raw(owner, repo, sha, item["path"], token) - (wf_dir / item["name"]).write_bytes(data) - print(f" {slug}/workflows/{item['name']}") - - return sha - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--token", default=os.environ.get("GITHUB_TOKEN")) - parser.add_argument("--slug", help="Fetch only this repo slug") - args = parser.parse_args() - - manifest = load_manifest() - repos = manifest.get("repos", []) - - if args.slug: - repos = [r for r in repos if r["slug"] == args.slug] - if not repos: - print(f"Unknown slug: {args.slug}", file=sys.stderr) - sys.exit(1) - - for entry in repos: - if entry.get("status") == "skip": - continue - print(f"Fetching {entry['slug']}...") - try: - fetch_repo(entry, args.token) - except Exception as e: - print(f" ERROR: {e}") - - -if __name__ == "__main__": - main() -``` - -**Step 2: Make executable** - -Run: `chmod +x proving-grounds/scripts/fetch-workflows.py` - -**Step 3: Commit** - -```bash -git add proving-grounds/scripts/fetch-workflows.py -git commit -m "feat: workflow fetch script for proving-grounds" -``` - ---- - -### Task 3: Write compile-tier test script - -**Files:** -- Create: `proving-grounds/scripts/compile-all.sh` - -**Step 1: Write the compile test script** - -This script iterates over all repos that have a `.harmont/pipeline.py`, runs `hm render`, and reports pass/fail. - -```bash -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPOS_DIR="$SCRIPT_DIR/../repos" -HM="${HM_BIN:-hm}" - -pass=0 -fail=0 -skip=0 -failures=() - -for repo_dir in "$REPOS_DIR"/*/; do - slug="$(basename "$repo_dir")" - pipeline="$repo_dir/.harmont/pipeline.py" - - if [[ ! -f "$pipeline" ]]; then - skip=$((skip + 1)) - continue - fi - - if $HM render ci --dir "$repo_dir" > /dev/null 2>&1; then - echo "PASS $slug" - pass=$((pass + 1)) - else - echo "FAIL $slug" - fail=$((fail + 1)) - failures+=("$slug") - fi -done - -echo "" -echo "── Results ──" -echo "pass: $pass fail: $fail skip: $skip" - -if [[ ${#failures[@]} -gt 0 ]]; then - echo "" - echo "Failures:" - for f in "${failures[@]}"; do - echo " - $f" - done - exit 1 -fi -``` - -**Step 2: Make executable** - -Run: `chmod +x proving-grounds/scripts/compile-all.sh` - -**Step 3: Commit** - -```bash -git add proving-grounds/scripts/compile-all.sh -git commit -m "feat: compile-tier test script for proving-grounds" -``` - ---- - -### Task 4: Write execution-tier test scripts - -**Files:** -- Create: `proving-grounds/scripts/execute.sh` -- Create: `proving-grounds/scripts/execute-all.sh` - -**Step 1: Write the single-repo execution script** - -```bash -#!/usr/bin/env bash -set -euo pipefail - -# Usage: execute.sh [--keep] -# Clones repo at pinned SHA, copies .harmont/, runs hm run ci. - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPOS_DIR="$SCRIPT_DIR/../repos" -HM="${HM_BIN:-hm}" -CACHE_DIR="${PROVING_GROUNDS_CACHE:-/tmp/proving-grounds}" - -slug="$1" -keep="${2:-}" -repo_dir="$REPOS_DIR/$slug" -meta="$repo_dir/metadata.toml" - -if [[ ! -f "$meta" ]]; then - echo "ERROR: no metadata.toml for $slug" >&2 - exit 1 -fi - -url=$(grep '^url' "$meta" | cut -d'"' -f2) -sha=$(grep '^sha' "$meta" | cut -d'"' -f2) - -if [[ -z "$sha" ]]; then - echo "ERROR: no pinned SHA for $slug" >&2 - exit 1 -fi - -clone_dir="$CACHE_DIR/$slug" - -if [[ -d "$clone_dir/.git" ]]; then - echo "Using cached clone: $clone_dir" - git -C "$clone_dir" checkout "$sha" 2>/dev/null || { - git -C "$clone_dir" fetch origin "$sha" - git -C "$clone_dir" checkout "$sha" - } -else - echo "Cloning $url @ $sha ..." - git clone --depth 1 "$url" "$clone_dir" 2>/dev/null || { - git clone "$url" "$clone_dir" - } - git -C "$clone_dir" checkout "$sha" 2>/dev/null || true -fi - -# Copy harmont pipeline into clone -if [[ -d "$repo_dir/.harmont" ]]; then - cp -r "$repo_dir/.harmont" "$clone_dir/.harmont" -fi - -echo "Running: $HM run ci --dir $clone_dir" -$HM run ci --dir "$clone_dir" -result=$? - -if [[ "$keep" != "--keep" ]]; then - rm -rf "$clone_dir" -fi - -exit $result -``` - -**Step 2: Write the batch execution script** - -```bash -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPOS_DIR="$SCRIPT_DIR/../repos" - -pass=0 -fail=0 -skip=0 -failures=() - -for repo_dir in "$REPOS_DIR"/*/; do - slug="$(basename "$repo_dir")" - - if [[ ! -d "$repo_dir/.harmont" ]]; then - skip=$((skip + 1)) - continue - fi - - echo "══════════════════════════════════════" - echo " $slug" - echo "══════════════════════════════════════" - - if "$SCRIPT_DIR/execute.sh" "$slug"; then - pass=$((pass + 1)) - else - fail=$((fail + 1)) - failures+=("$slug") - fi - echo "" -done - -echo "" -echo "── Execution Results ──" -echo "pass: $pass fail: $fail skip: $skip" - -if [[ ${#failures[@]} -gt 0 ]]; then - echo "" - echo "Failures:" - for f in "${failures[@]}"; do - echo " - $f" - done - exit 1 -fi -``` - -**Step 3: Make executable** - -Run: -```bash -chmod +x proving-grounds/scripts/execute.sh proving-grounds/scripts/execute-all.sh -``` - -**Step 4: Commit** - -```bash -git add proving-grounds/scripts/execute.sh proving-grounds/scripts/execute-all.sh -git commit -m "feat: execution-tier test scripts for proving-grounds" -``` - ---- - -### Task 5: Write Rust integration test for compile tier - -**Files:** -- Create: `proving-grounds/tests/compile_fixtures.rs` -- Modify: `cli/Cargo.toml` (add proving-grounds test target if needed) - -**Step 1: Write the Rust compile test** - -This test auto-discovers all repos with `.harmont/pipeline.py` and validates their IR. - -```rust -#![allow(clippy::unwrap_used, clippy::expect_used)] - -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -fn proving_grounds_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../proving-grounds/repos") -} - -fn hm_bin() -> String { - std::env::var("HM_BIN").unwrap_or_else(|_| "hm".to_string()) -} - -fn repo_slugs() -> Vec { - let dir = proving_grounds_dir(); - if !dir.exists() { - return vec![]; - } - let mut slugs = vec![]; - for entry in fs::read_dir(&dir).unwrap() { - let entry = entry.unwrap(); - let harmont = entry.path().join(".harmont/pipeline.py"); - if harmont.exists() { - slugs.push(entry.file_name().to_string_lossy().to_string()); - } - } - slugs.sort(); - slugs -} - -#[test] -fn all_proving_ground_fixtures_render() { - let slugs = repo_slugs(); - if slugs.is_empty() { - eprintln!("SKIP: no proving-ground fixtures with .harmont/ found"); - return; - } - - let mut failures = vec![]; - - for slug in &slugs { - let dir = proving_grounds_dir().join(slug); - let output = Command::new(hm_bin()) - .args(["render", "ci", "--dir"]) - .arg(&dir) - .output() - .unwrap_or_else(|e| panic!("spawn hm: {e}")); - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - failures.push(format!("{slug}: {stderr}")); - continue; - } - - // Validate JSON parses - let stdout = String::from_utf8_lossy(&output.stdout); - let _: serde_json::Value = serde_json::from_str(&stdout) - .unwrap_or_else(|e| panic!("{slug}: invalid JSON: {e}")); - } - - if !failures.is_empty() { - panic!( - "{} fixture(s) failed to render:\n{}", - failures.len(), - failures.join("\n") - ); - } -} - -#[test] -fn all_proving_ground_fixtures_have_valid_structure() { - let slugs = repo_slugs(); - if slugs.is_empty() { - return; - } - - for slug in &slugs { - let dir = proving_grounds_dir().join(slug); - let output = Command::new(hm_bin()) - .args(["render", "ci", "--dir"]) - .arg(&dir) - .output() - .unwrap(); - - if !output.status.success() { - continue; // caught by render test above - } - - let stdout = String::from_utf8_lossy(&output.stdout); - let ir: serde_json::Value = serde_json::from_str(&stdout).unwrap(); - - // Validate v0 IR structure - assert_eq!(ir["version"], "0", "{slug}: version != 0"); - - let nodes = ir["graph"]["nodes"].as_array() - .unwrap_or_else(|| panic!("{slug}: missing graph.nodes")); - assert!(!nodes.is_empty(), "{slug}: no nodes"); - - for node in nodes { - let key = node["step"]["key"].as_str() - .unwrap_or_else(|| panic!("{slug}: node missing step.key")); - assert!(!key.is_empty(), "{slug}: empty key"); - - let cmd = node["step"]["cmd"].as_str() - .unwrap_or_else(|| panic!("{slug}: node missing step.cmd")); - assert!(!cmd.is_empty(), "{slug}: empty cmd for key {key}"); - } - - let edges = ir["graph"]["edges"].as_array() - .unwrap_or_else(|| panic!("{slug}: missing graph.edges")); - assert!(!edges.is_empty(), "{slug}: no edges"); - - // No self-loops - for edge in edges { - let src = edge[0].as_u64().unwrap(); - let dst = edge[1].as_u64().unwrap(); - assert_ne!(src, dst, "{slug}: self-loop on node {src}"); - } - } -} -``` - -**Step 2: Run test to verify it compiles (will skip if no fixtures yet)** - -Run: `cargo test -p hm --test compile_fixtures -- --nocapture` -Expected: SKIP message (no fixtures yet) - -**Step 3: Commit** - -```bash -git add proving-grounds/tests/compile_fixtures.rs -git commit -m "feat: compile-tier Rust integration test for proving-grounds" -``` - ---- - -### Task 6: Write pipeline rewrite — BurntSushi/ripgrep (Rust, simple) - -**Files:** -- Create: `proving-grounds/repos/BurntSushi--ripgrep/.harmont/pipeline.py` - -**Step 1: Study the original CI** - -Run: `python proving-grounds/scripts/fetch-workflows.py --slug BurntSushi--ripgrep` -Then read the downloaded workflow YAML to understand what the CI does. - -**Step 2: Write the harmont rewrite** - -```python -"""ripgrep — Rust CLI for recursive regex search.""" -from __future__ import annotations - -import harmont as hm -from harmont.rust import RustToolchain - - -@hm.target() -def project() -> RustToolchain: - return hm.rust.toolchain(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.clippy(), - project.fmt(), - ) -``` - -**Step 3: Verify compile** - -Run: `hm render ci --dir proving-grounds/repos/BurntSushi--ripgrep` -Expected: Valid JSON IR output - -**Step 4: Commit** - -```bash -git add proving-grounds/repos/BurntSushi--ripgrep/ -git commit -m "feat(proving-grounds): add ripgrep pipeline rewrite" -``` - ---- - -### Task 7: Write pipeline rewrite — pallets/flask (Python, simple) - -**Files:** -- Create: `proving-grounds/repos/pallets--flask/.harmont/pipeline.py` - -**Step 1: Fetch and study original CI** - -Run: `python proving-grounds/scripts/fetch-workflows.py --slug pallets--flask` - -**Step 2: Write the harmont rewrite** - -```python -"""Flask — lightweight Python web framework.""" -from __future__ import annotations - -import harmont as hm -from harmont.python import PythonToolchain - - -@hm.target() -def project() -> PythonToolchain: - return hm.python(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: - return ( - project.test(), - project.lint(), - project.typecheck(), - ) -``` - -**Step 3: Verify compile** - -Run: `hm render ci --dir proving-grounds/repos/pallets--flask` - -**Step 4: Commit** - -```bash -git add proving-grounds/repos/pallets--flask/ -git commit -m "feat(proving-grounds): add flask pipeline rewrite" -``` - ---- - -### Task 8: Write pipeline rewrite — expressjs/express (JS, simple) - -**Files:** -- Create: `proving-grounds/repos/expressjs--express/.harmont/pipeline.py` - -**Step 1: Fetch and study original CI** - -Run: `python proving-grounds/scripts/fetch-workflows.py --slug expressjs--express` - -**Step 2: Write the harmont rewrite** - -```python -"""Express — minimal Node.js web framework.""" -from __future__ import annotations - -import harmont as hm -from harmont.npm import NpmProject - - -@hm.target() -def project() -> NpmProject: - return hm.npm(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: - return ( - project.test(), - project.lint(), - ) -``` - -**Step 3: Verify compile, commit** - ---- - -### Task 9: Write pipeline rewrite — gin-gonic/gin (Go, simple) - -**Files:** -- Create: `proving-grounds/repos/gin-gonic--gin/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Gin — HTTP web framework for Go.""" -from __future__ import annotations - -import harmont as hm -from harmont.go import GoToolchain - - -@hm.target() -def project() -> GoToolchain: - return hm.go(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[GoToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.vet(), - project.fmt(), - ) -``` - -**Step 3: Verify compile, commit** - ---- - -### Task 10: Write pipeline rewrite — clap-rs/clap (Rust, simple) - -**Files:** -- Create: `proving-grounds/repos/clap-rs--clap/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""clap — CLI argument parser for Rust.""" -from __future__ import annotations - -import harmont as hm -from harmont.rust import RustToolchain - - -@hm.target() -def project() -> RustToolchain: - return hm.rust.toolchain(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.clippy(), - project.fmt(), - ) -``` - ---- - -### Task 11: Write pipeline rewrite — tokio-rs/tokio (Rust, medium) - -**Files:** -- Create: `proving-grounds/repos/tokio-rs--tokio/.harmont/pipeline.py` - -Tokio is a Cargo workspace with multiple crates. The pipeline needs to build/test the workspace, plus run clippy and fmt. Harmont's Rust toolchain operates at workspace level. - -**Step 2: Write the harmont rewrite** - -```python -"""Tokio — async runtime for Rust.""" -from __future__ import annotations - -import harmont as hm -from harmont.rust import RustToolchain - - -@hm.target() -def project() -> RustToolchain: - return hm.rust.toolchain(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true", "RUSTFLAGS": "-Dwarnings"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.clippy(), - project.fmt(), - ) -``` - ---- - -### Task 12: Write pipeline rewrite — vitejs/vite (TypeScript, medium) - -**Files:** -- Create: `proving-grounds/repos/vitejs--vite/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Vite — next-generation frontend build tool.""" -from __future__ import annotations - -import harmont as hm -from harmont.npm import NpmProject - - -@hm.target() -def project() -> NpmProject: - return hm.npm(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: - return ( - project.run("build"), - project.run("test"), - project.run("lint"), - ) -``` - ---- - -### Task 13: Write pipeline rewrite — hashicorp/terraform (Go, medium) - -**Files:** -- Create: `proving-grounds/repos/hashicorp--terraform/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Terraform — infrastructure as code.""" -from __future__ import annotations - -import harmont as hm -from harmont.go import GoToolchain - - -@hm.target() -def project() -> GoToolchain: - return hm.go(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[GoToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.vet(), - project.fmt(), - ) -``` - ---- - -### Task 14: Write pipeline rewrite — django/django (Python, medium) - -**Files:** -- Create: `proving-grounds/repos/django--django/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Django — Python web framework.""" -from __future__ import annotations - -import harmont as hm -from harmont.python import PythonToolchain - - -@hm.target() -def project() -> PythonToolchain: - return hm.python(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: - return ( - project.test(), - project.lint(), - project.fmt(), - ) -``` - ---- - -### Task 15: Write pipeline rewrite — astral-sh/ruff (Rust, medium) - -**Files:** -- Create: `proving-grounds/repos/astral-sh--ruff/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Ruff — fast Python linter/formatter written in Rust.""" -from __future__ import annotations - -import harmont as hm -from harmont.rust import RustToolchain - - -@hm.target() -def project() -> RustToolchain: - return hm.rust.toolchain(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.clippy(), - project.fmt(), - ) -``` - ---- - -### Task 16: Write pipeline rewrite — jekyll/jekyll (Ruby, simple) - -**Files:** -- Create: `proving-grounds/repos/jekyll--jekyll/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Jekyll — static site generator in Ruby.""" -from __future__ import annotations - -import harmont as hm -from harmont.ruby import RubyToolchain - - -@hm.target() -def project() -> RubyToolchain: - return hm.ruby(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[RubyToolchain]) -> tuple[hm.Step, ...]: - return ( - project.test(), - project.lint(), - ) -``` - ---- - -### Task 17: Write pipeline rewrite — starship/starship (Rust, medium) - -**Files:** -- Create: `proving-grounds/repos/starship--starship/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Starship — cross-shell prompt.""" -from __future__ import annotations - -import harmont as hm -from harmont.rust import RustToolchain - - -@hm.target() -def project() -> RustToolchain: - return hm.rust.toolchain(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.clippy(), - project.fmt(), - ) -``` - ---- - -### Task 18: Write pipeline rewrite — grafana/grafana (Go+TS, complex monorepo) - -**Files:** -- Create: `proving-grounds/repos/grafana--grafana/.harmont/pipeline.py` - -Grafana is a Go backend + TypeScript frontend monorepo. This demonstrates harmont's multi-toolchain composition — the same pattern as the `zig-js` example. - -**Step 2: Write the harmont rewrite** - -```python -"""Grafana — observability platform (Go + TypeScript monorepo).""" -from __future__ import annotations - -from typing import Annotated - -import harmont as hm -from harmont.go import GoToolchain -from harmont.npm import NpmProject - - -@hm.target() -def backend() -> GoToolchain: - return hm.go(path=".") - - -@hm.target() -def frontend() -> NpmProject: - return hm.npm(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci( - backend: hm.Target[GoToolchain], - frontend: hm.Target[NpmProject], -) -> tuple[hm.Step, ...]: - return ( - # Go backend - backend.build(), - backend.test(), - backend.vet(), - # TypeScript frontend - frontend.run("build"), - frontend.run("test"), - frontend.run("lint"), - ) -``` - ---- - -### Task 19: Write pipeline rewrite — fastapi/fastapi (Python, simple) - -**Files:** -- Create: `proving-grounds/repos/fastapi--fastapi/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""FastAPI — modern Python web framework.""" -from __future__ import annotations - -import harmont as hm -from harmont.python import PythonToolchain - - -@hm.target() -def project() -> PythonToolchain: - return hm.python(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="master")], -) -def ci(project: hm.Target[PythonToolchain]) -> tuple[hm.Step, ...]: - return ( - project.test(), - project.lint(), - project.fmt(), - project.typecheck(), - ) -``` - ---- - -### Task 20: Write pipeline rewrite — sveltejs/svelte (TypeScript, medium) - -**Files:** -- Create: `proving-grounds/repos/sveltejs--svelte/.harmont/pipeline.py` - -**Step 2: Write the harmont rewrite** - -```python -"""Svelte — cybernetically enhanced web apps.""" -from __future__ import annotations - -import harmont as hm -from harmont.npm import NpmProject - - -@hm.target() -def project() -> NpmProject: - return hm.npm(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[NpmProject]) -> tuple[hm.Step, ...]: - return ( - project.run("build"), - project.run("test"), - project.run("lint"), - ) -``` - ---- - -### Task 21: Batch commit all pipeline rewrites - -**Step 1: Verify all fixtures compile** - -Run: `proving-grounds/scripts/compile-all.sh` -Expected: All 15 PASS - -**Step 2: Commit all rewrites** - -```bash -git add proving-grounds/repos/ -git commit -m "feat(proving-grounds): initial 15 pipeline rewrites - -Repos: ripgrep, flask, express, gin, clap, tokio, vite, terraform, -django, ruff, jekyll, starship, grafana, fastapi, svelte" -``` - ---- - -### Task 22: Add CI integration for compile tier - -**Files:** -- Modify: `.github/workflows/ci.yml` (or create proving-grounds-specific workflow) - -**Step 1: Add proving-grounds compile step to CI** - -Add a job to the CI workflow: - -```yaml - proving-grounds-compile: - name: Proving Grounds (compile) - runs-on: ubuntu-latest - needs: [build] # depends on hm binary being built - steps: - - uses: actions/checkout@v4 - - name: Download hm binary - uses: actions/download-artifact@v4 - with: - name: hm-linux - path: /usr/local/bin/ - - run: chmod +x /usr/local/bin/hm - - name: Install Python deps - run: | - pip install harmont - - name: Compile all proving-ground fixtures - run: proving-grounds/scripts/compile-all.sh -``` - -**Step 2: Commit** - -```bash -git add .github/workflows/ -git commit -m "ci: add proving-grounds compile tier to CI" -``` - ---- - -### Task 23: Add nightly execution tier workflow - -**Files:** -- Create: `.github/workflows/proving-grounds-nightly.yml` - -**Step 1: Write the nightly workflow** - -```yaml -name: Proving Grounds (nightly) - -on: - schedule: - - cron: '0 4 * * *' # 4am UTC daily - workflow_dispatch: - -jobs: - execute: - name: Execute proving-ground pipelines - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: Build hm - run: cargo build --release -p hm - - name: Install Python deps - run: pip install harmont - - name: Run execution tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HM_BIN: ./target/release/hm - run: proving-grounds/scripts/execute-all.sh -``` - -**Step 2: Commit** - -```bash -git add .github/workflows/proving-grounds-nightly.yml -git commit -m "ci: add nightly proving-grounds execution workflow" -``` - ---- - -## Expansion Guide: Adding More Repos - -For each new repo (remaining 85 of 100): - -1. **Verify it's in `manifest.toml`** — entry exists with correct metadata -2. **Fetch workflows**: `python proving-grounds/scripts/fetch-workflows.py --slug ` -3. **Study the original CI** — read the downloaded YAML in `repos//workflows/` -4. **Write `.harmont/pipeline.py`** — follow the pattern from existing rewrites: - - Pick the right toolchain(s) for the repo's language - - Express the core pipeline: build → test → lint → fmt - - For monorepos, use multi-toolchain composition (Task 18 pattern) - - For repos with custom build steps, use `sh()` directly -5. **Verify compile**: `hm render ci --dir proving-grounds/repos/` -6. **Update manifest status**: Change `status = "pending"` to `"compile-only"` or `"passing"` -7. **Commit** - -### When a repo's build needs custom steps - -Some repos need steps beyond what toolchains provide. Use `sh()`: - -```python -@hm.pipeline("ci", default_image="ubuntu:24.04") -def ci(project: hm.Target[RustToolchain]) -> tuple[hm.Step, ...]: - installed = project.installed - custom = installed.sh("cargo xtask codegen", label=":codegen:") - return ( - custom, - project.build(), - project.test(), - ) -``` - -### What harmont currently CANNOT express - -These features appear in real-world CI but have no harmont equivalent yet. When rewriting, express only what harmont supports: - -| GHA Feature | Harmont Status | Workaround | -|-------------|---------------|------------| -| Matrix builds | Not supported | Write single-config pipeline (one OS, one version) | -| Conditional steps (`if:`) | Not supported | Include unconditionally or omit | -| Artifact upload/download | Not needed | Harmont uses snapshot lineage | -| Reusable workflows | Not supported | Inline the logic | -| Self-hosted runners | Not supported | Use default `ubuntu:24.04` image | -| Service containers | Not supported | Install services in pipeline steps | - -### Priority order for remaining 85 repos - -1. **Simple repos with matching toolchains** — bat, fd, zoxide, serde, requests, black, poetry, got, chalk, prettier, chi, testify, fzf (quick wins, ~5 min each) -2. **Medium repos with matching toolchains** — nushell, helix, ruff, pnpm, deno, prometheus, vault, etcd (slightly more complex but same pattern) -3. **Complex repos** — these may surface DSL gaps; each needs careful study of original CI -4. **Extreme repos** — mostly reference/aspiration; many use non-GHA CI systems - ---- - -## Success Metrics - -| Metric | Target | -|--------|--------| -| Repos with compile-only status | 80/100 within 2 weeks | -| Repos with passing execution | 30/100 within 1 month | -| Compile tier CI green | Every commit | -| Execution tier nightly green | 80% of repos with pipelines | -| DSL gaps identified | Documented per repo in metadata.toml | diff --git a/proving-grounds/.harmont/pipeline.py b/proving-grounds/.harmont/pipeline.py index f4ac3ace..70c4b2e7 100644 --- a/proving-grounds/.harmont/pipeline.py +++ b/proving-grounds/.harmont/pipeline.py @@ -1,8 +1,11 @@ """Proving grounds — exercise harmont's DSL against real-world OSS repos. -Each repo is cloned into /opt/ and built/tested/linted using -harmont's toolchain abstractions. All repos run as parallel chains +Each repo is cloned at a pinned SHA into /opt/ and built/tested/linted +using harmont's toolchain abstractions. All repos run as parallel chains from a shared apt base. + +Repos are audited individually — each uses the exact build/test/lint +commands from its real CI, not generic defaults. """ from __future__ import annotations @@ -11,34 +14,20 @@ import harmont as hm -RUST_REPOS = [ - ("ripgrep", "https://github.com/BurntSushi/ripgrep"), - ("clap", "https://github.com/clap-rs/clap"), - ("tokio", "https://github.com/tokio-rs/tokio"), - ("starship", "https://github.com/starship/starship"), - ("ruff", "https://github.com/astral-sh/ruff"), -] - -PYTHON_REPOS = [ - ("flask", "https://github.com/pallets/flask"), - ("django", "https://github.com/django/django"), - ("fastapi", "https://github.com/fastapi/fastapi"), -] - -NPM_REPOS = [ - ("express", "https://github.com/expressjs/express"), - ("vite", "https://github.com/vitejs/vite"), - ("svelte", "https://github.com/sveltejs/svelte"), -] - -GO_REPOS = [ - ("gin", "https://github.com/gin-gonic/gin"), - ("terraform", "https://github.com/hashicorp/terraform"), -] - -RUBY_REPOS = [ - ("jekyll", "https://github.com/jekyll/jekyll"), -] +# (name, url, sha) +REPOS = { + "ripgrep": ("https://github.com/BurntSushi/ripgrep", "82313cf95849"), + "clap": ("https://github.com/clap-rs/clap", "8387c812c4b9"), + "tokio": ("https://github.com/tokio-rs/tokio", "778e9d97d91c"), + "starship": ("https://github.com/starship/starship", "f595899021ca"), + "ruff": ("https://github.com/astral-sh/ruff", "8cf0c77eb2be"), + "flask": ("https://github.com/pallets/flask", "36e4a824f340"), + "fastapi": ("https://github.com/fastapi/fastapi", "5cdf820c8046"), + "express": ("https://github.com/expressjs/express", "dae209ae6559"), + "gin": ("https://github.com/gin-gonic/gin", "d75fcd4c9ab2"), + "terraform": ("https://github.com/hashicorp/terraform", "d038b9e6cd86"), + "jekyll": ("https://github.com/jekyll/jekyll", "202df571314b"), +} @hm.target() @@ -46,47 +35,190 @@ def apt(root: Annotated[hm.Step, hm.BaseImage("ubuntu:24.04")]) -> hm.Step: return root.sh( "apt-get update && " "apt-get install -y --no-install-recommends " - "git ca-certificates curl build-essential pkg-config", + "git ca-certificates curl build-essential pkg-config " + "libdbus-1-dev", label=":apt: base", cache=hm.ttl(timedelta(days=1)), ) -def _clone(base: hm.Step, name: str, url: str) -> hm.Step: +def _clone(base: hm.Step, name: str) -> hm.Step: + url, sha = REPOS[name] return base.fork(name).sh( - f"git clone --depth 1 {url} /opt/{name}", + f"git init /opt/{name} && " + f"git -C /opt/{name} fetch --depth 1 {url} {sha} && " + f"git -C /opt/{name} checkout FETCH_HEAD", label=f":git: {name}", ) -def _rust_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: - cloned = _clone(apt, name, url) - tc = hm.rust.toolchain(path=f"/opt/{name}", base=cloned) - return [tc.build(), tc.test(), tc.clippy(), tc.fmt()] - - -def _python_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: - cloned = _clone(apt, name, url) - tc = hm.python(path=f"/opt/{name}", base=cloned) - return [tc.test(), tc.lint(), tc.typecheck()] - - -def _npm_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: - cloned = _clone(apt, name, url) - project = hm.npm(path=f"/opt/{name}", base=cloned) - return [project.run("build"), project.run("test"), project.run("lint")] +def _rust(base: hm.Step, name: str, **kw: str) -> hm.Step: + """Install Rust toolchain on top of a cloned repo.""" + version = kw.get("version", "stable") + cloned = _clone(base, name) + return hm.rust.toolchain( + path=f"/opt/{name}", base=cloned, version=version, + ).installed + + +# ── Rust: ripgrep ──────────────────────────────────────────────────── +# No clippy in their CI. Simple cargo build/test/fmt. +def _ripgrep(apt: hm.Step) -> list[hm.Step]: + tc = _rust(apt, "ripgrep") + cd = f". $HOME/.cargo/env && cd /opt/ripgrep" + return [ + tc.sh(f"{cd} && cargo build --workspace", label=":ripgrep: build"), + tc.sh(f"{cd} && cargo test --workspace", label=":ripgrep: test"), + tc.sh(f"{cd} && cargo fmt --all --check", label=":ripgrep: fmt"), + ] + + +# ── Rust: clap ─────────────────────────────────────────────────────── +# Needs feature flags and -A deprecated for clippy. +def _clap(apt: hm.Step) -> list[hm.Step]: + tc = _rust(apt, "clap") + cd = f". $HOME/.cargo/env && cd /opt/clap" + features = "deprecated derive cargo env unicode string wrap_help unstable-ext" + return [ + tc.sh(f"{cd} && cargo build --workspace --features \"{features}\" --all-targets", + label=":clap: build"), + tc.sh(f"{cd} && cargo test --workspace --features \"{features}\"", + label=":clap: test"), + tc.sh(f"{cd} && cargo clippy --workspace --all-targets " + f"--features \"{features}\" -- -D warnings -A deprecated", + label=":clap: clippy"), + tc.sh(f"{cd} && cargo fmt --all --check", label=":clap: fmt"), + ] + + +# ── Rust: tokio ────────────────────────────────────────────────────── +# Needs --features full,test-util. Uses rustfmt directly (not cargo fmt). +def _tokio(apt: hm.Step) -> list[hm.Step]: + tc = _rust(apt, "tokio") + cd = f". $HOME/.cargo/env && cd /opt/tokio" + env = "RUSTFLAGS='-Dwarnings'" + return [ + tc.sh(f"{cd} && {env} cargo build --workspace --features full,test-util", + label=":tokio: build"), + tc.sh(f"{cd} && {env} cargo test --workspace --features full,test-util", + label=":tokio: test"), + tc.sh(f"{cd} && cargo clippy --workspace --tests --no-deps " + f"--features full,test-util -- -D warnings", + label=":tokio: clippy"), + tc.sh(f"{cd} && rustfmt --check --edition 2021 $(git ls-files '*.rs')", + label=":tokio: fmt"), + ] + + +# ── Rust: starship ─────────────────────────────────────────────────── +# Needs --locked. libdbus-1-dev already in apt base. +def _starship(apt: hm.Step) -> list[hm.Step]: + tc = _rust(apt, "starship") + cd = f". $HOME/.cargo/env && cd /opt/starship" + return [ + tc.sh(f"{cd} && cargo build --workspace --locked", label=":starship: build"), + tc.sh(f"{cd} && cargo test --workspace --locked", label=":starship: test"), + tc.sh(f"{cd} && cargo clippy --workspace --locked -- -D warnings", + label=":starship: clippy"), + tc.sh(f"{cd} && cargo fmt --all -- --check", label=":starship: fmt"), + ] + + +# ── Rust: ruff ─────────────────────────────────────────────────────── +# Pinned to Rust 1.96 via rust-toolchain.toml. Needs --all-features --locked. +def _ruff(apt: hm.Step) -> list[hm.Step]: + tc = _rust(apt, "ruff", version="1.96.0") + cd = f". $HOME/.cargo/env && cd /opt/ruff" + return [ + tc.sh(f"{cd} && cargo build --workspace --locked", label=":ruff: build"), + tc.sh(f"{cd} && cargo test --workspace --all-features --locked", + label=":ruff: test"), + tc.sh(f"{cd} && cargo clippy --workspace --all-targets --all-features " + f"--locked -- -D warnings", + label=":ruff: clippy"), + tc.sh(f"{cd} && cargo fmt --all --check", label=":ruff: fmt"), + ] + + +# ── Python: flask ──────────────────────────────────────────────────── +# Native uv project. Uses ruff + ty. Standard hm.python() works. +def _flask(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "flask") + tc = hm.python(path="/opt/flask", base=cloned) + return [tc.test(), tc.lint(), tc.fmt(), tc.typecheck()] + + +# ── Python: fastapi ────────────────────────────────────────────────── +# Uses uv but needs custom sync flags and test invocation. +def _fastapi(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "fastapi") + tc = hm.python(path="/opt/fastapi", base=cloned) + cd = "cd /opt/fastapi" + return [ + tc.installed.sh( + f"{cd} && PYTHONPATH=./docs_src uv run pytest tests/ -x -q", + label=":fastapi: test", + ), + tc.lint(), + tc.fmt(), + tc.installed.sh( + f"{cd} && uv run ty check fastapi", + label=":fastapi: typecheck", + ), + ] + + +# ── npm: express ───────────────────────────────────────────────────── +# Plain npm. No build script. Just test + lint. +def _express(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "express") + project = hm.npm(path="/opt/express", base=cloned) + return [project.test(), project.lint()] -def _go_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: - cloned = _clone(apt, name, url) - tc = hm.go(path=f"/opt/{name}", base=cloned) +# ── Go: gin ────────────────────────────────────────────────────────── +# Simple Go project. Standard commands work. +def _gin(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "gin") + tc = hm.go(path="/opt/gin", base=cloned) return [tc.build(), tc.test(), tc.vet(), tc.fmt()] -def _ruby_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: - cloned = _clone(apt, name, url) - project = hm.ruby(path=f"/opt/{name}", base=cloned) - return [project.test(), project.lint()] +# ── Go: terraform ──────────────────────────────────────────────────── +# Multi-module workspace. Test all sub-modules via loop. +def _terraform(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "terraform") + tc = hm.go(path="/opt/terraform", base=cloned) + cd = "cd /opt/terraform && export PATH=$HOME/go/bin:$PATH" + return [ + tc.build(), + tc.installed.sh( + f"{cd} && " + "for dir in $(go list -m -f '{{{{.Dir}}}}' github.com/hashicorp/terraform/...); do " + "(cd $dir && go test -short -count=1 ./...) || exit 1; done", + label=":terraform: test", + ), + tc.vet(), + tc.fmt(), + ] + + +# ── Ruby: jekyll ───────────────────────────────────────────────────── +# Bundler project. Tests need TZ=UTC. Lint via rubocop. +def _jekyll(apt: hm.Step) -> list[hm.Step]: + cloned = _clone(apt, "jekyll") + project = hm.ruby(path="/opt/jekyll", base=cloned) + cd = "cd /opt/jekyll" + return [ + project.installed.sh( + f"{cd} && TZ=UTC bundle exec rake test", + label=":jekyll: test", + ), + project.installed.sh( + f"{cd} && bundle exec rubocop -D --disable-pending-cops", + label=":jekyll: lint", + ), + ] @hm.pipeline( @@ -97,32 +229,16 @@ def _ruby_leaves(apt: hm.Step, name: str, url: str) -> list[hm.Step]: def ci(apt: hm.Target[hm.Step]) -> tuple[hm.Step, ...]: leaves: list[hm.Step] = [] - for name, url in RUST_REPOS: - leaves.extend(_rust_leaves(apt, name, url)) - - for name, url in PYTHON_REPOS: - leaves.extend(_python_leaves(apt, name, url)) - - for name, url in NPM_REPOS: - leaves.extend(_npm_leaves(apt, name, url)) - - for name, url in GO_REPOS: - leaves.extend(_go_leaves(apt, name, url)) - - for name, url in RUBY_REPOS: - leaves.extend(_ruby_leaves(apt, name, url)) - - # Grafana — multi-toolchain monorepo (Go + npm) - grafana_clone = _clone(apt, "grafana", "https://github.com/grafana/grafana") - grafana_go = hm.go(path="/opt/grafana", base=grafana_clone) - grafana_npm = hm.npm(path="/opt/grafana", base=grafana_clone) - leaves.extend([ - grafana_go.build(), - grafana_go.test(), - grafana_go.vet(), - grafana_npm.run("build"), - grafana_npm.run("test"), - grafana_npm.run("lint"), - ]) + leaves.extend(_ripgrep(apt)) + leaves.extend(_clap(apt)) + leaves.extend(_tokio(apt)) + leaves.extend(_starship(apt)) + leaves.extend(_ruff(apt)) + leaves.extend(_flask(apt)) + leaves.extend(_fastapi(apt)) + leaves.extend(_express(apt)) + leaves.extend(_gin(apt)) + leaves.extend(_terraform(apt)) + leaves.extend(_jekyll(apt)) return tuple(leaves) From 6db0e713e86d2e7c5520fd63d3d7c542e4f76845 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 6 Jun 2026 15:20:27 -0700 Subject: [PATCH 3/3] fix: remove dead test assertions after toolchain removal - kitchen-sink fixture now has 10 nodes (c + ruby), not 12+ (had haskell + cmake + c). Update node count assertions and replace haskell label check with ruby. - Remove stale elm reference in _toolchain.py comment. --- crates/hm-dsl-engine/harmont-py/harmont/_toolchain.py | 3 +-- crates/hm-pipeline-ir/tests/e2e_fixtures.rs | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_toolchain.py b/crates/hm-dsl-engine/harmont-py/harmont/_toolchain.py index e3afe269..a3a5cd70 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_toolchain.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_toolchain.py @@ -36,8 +36,7 @@ def apt_install_cmd(packages: tuple[str, ...]) -> str: def node_install_cmd(version: str) -> str: """NodeSource node-install command for a given major Node version. - Used by both the npm toolchain and the elm toolchain (whose - tooling runs under npx). + Used by the npm toolchain. """ major = version.removesuffix(".x") return ( diff --git a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs b/crates/hm-pipeline-ir/tests/e2e_fixtures.rs index 74d839d5..9b08528d 100644 --- a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs +++ b/crates/hm-pipeline-ir/tests/e2e_fixtures.rs @@ -96,9 +96,9 @@ fn python_zig_node_polyglot() { fn python_kitchen_sink() { let g = load_fixture("python", "kitchen-sink"); assert_eq!(g.default_image(), Some("ubuntu:24.04")); - assert!(g.node_count() >= 12, "nodes: {}", g.node_count()); + assert!(g.node_count() >= 10, "nodes: {}", g.node_count()); let labels = step_labels(&g); - assert!(labels.iter().any(|l| l.contains("haskell"))); + assert!(labels.iter().any(|l| l.contains("ruby"))); assert!( labels .iter() @@ -138,7 +138,7 @@ fn ts_zig_node_polyglot() { fn ts_kitchen_sink() { let g = load_fixture("ts", "kitchen-sink"); assert_eq!(g.default_image(), Some("ubuntu:24.04")); - assert!(g.node_count() >= 12); + assert!(g.node_count() >= 10); } #[test]