diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4265b26..27f0708 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -25,3 +25,18 @@ jobs: - run: python -m unittest discover -s tests -v - run: python -m com_jepa validate examples/fictional-trajectory.jsonl - run: python -m com_jepa snapshot examples/fictional-trajectory.jsonl --as-of 2026-01-06T12:00:00Z + + forest-demo: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + - run: python -m pip install -r requirements-ml.txt + - run: python -m unittest discover -s tests -v + - run: python -m com_jepa validate examples/fictional-trajectory.jsonl + - run: python -m com_jepa.forest_demo --output artifacts/forest-demo.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aa96f3..7dc932e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased — initial research scaffold +- Add an optional scikit-learn Random Forest demo on explicitly invented tabular snapshots, with a training-only base-rate comparator, temporal label eligibility, exclusion accounting, reproducible reports, and ML CI. No real organisational findings are implied. - Establish the organisational thesis, claims ledger, and baseline-first research plan. - Propose a draft `0.1.0` commitment event export with version lineage and occurrence/availability timestamps. - Add a wholly fictional trajectory, context projection utility, and temporal/semantic validation tests. diff --git a/README.md b/README.md index 576563f..36714c6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Today, people often carry the missing connections between business systems. They **com-jepa asks whether a history of commitments, decisions, actions, and observed consequences can improve the next organisational decision.** It starts with an explicit data contract and simple statistical models. It will investigate Joint Embedding Predictive Architectures (JEPA) if the evidence and data justify that step. -This is an open research project initiated by [Reflective Lab](https://www.reflective.se). It is at the research-design and data-contract stage. There is no trained organisational JEPA model, production predictor, or empirical performance claim in this repository. The included data is entirely fictional. +This is an open research project initiated by [Reflective Lab](https://www.reflective.se). It is at the research-design and data-contract stage, with an executable Random Forest demonstration trained on invented rows. There is no trained organisational JEPA model, production predictor, or empirical organisational performance claim in this repository. The included data is entirely fictional. [Reflective research library](https://www.reflective.se/labs/research) · [Research plan](docs/research-plan.md) · [Data contract](docs/data-contract.md) · [Contribute](CONTRIBUTING.md) · [Security](SECURITY.md) @@ -68,6 +68,8 @@ A simple model that wins is a successful research result. A shared model valid f | [Foundations](docs/foundations.md) | The organisation, human agency, and learning from priors | | [Research plan](docs/research-plan.md) | Hypotheses, comparisons, evaluation, and stop conditions | | [Data contract](docs/data-contract.md) | What to collect, label, connect, and keep out | +| [Random Forest demo](docs/random-forest-demo.md) | A small CPU-only classifier, historical base-rate comparison, and temporal checks on synthetic data | +| [TabPFN-3 comparison](docs/tabpfn-comparison.md) | An optional GPU experiment using the same rows and a pinned pretrained classifier | | [Application integration](docs/application-integration.md) | How apps contribute to the Organisation Core | | [First 90 days](docs/first-90-days.md) | A bounded starting project and collaboration questions | | [Reading list](docs/reading-list.md) | Intellectual lineage and the limits of the evidence | @@ -90,11 +92,22 @@ The validation command reports **11 valid fictional events**. The Tuesday-noon s ## Project status +To try the optional ML example, use Python 3.14 and the pinned scikit-learn dependencies: + +```sh +python3.14 -m venv .venv +.venv/bin/python -m pip install -r requirements-ml.txt +.venv/bin/python -m com_jepa.forest_demo +``` + +This fits a Random Forest to 300 invented commitment snapshots and compares it with the training fulfilment rate on a later test period. Unknown/disputed/censored labels are excluded, and training labels must be available before fitting. See the [demo guide](docs/random-forest-demo.md) for the exact synthetic rule, JSON reports, and limitations. PyTorch and a GPU are not required. + | Available now | Proposed next | | --- | --- | | Organisational thesis and falsifiable research questions | Practitioner review and selection of one commitment family | | Draft JSON Schema, fictional trajectory, and historical context utility | A governed prospective pilot and real outcome adjudication | | Temporal and lineage tests; GitHub CI | A reproducible benchmark with conventional ML baselines | +| Synthetic Random Forest example and base-rate comparator | Real-data feature extraction, outcome adjudication, and validation | | Evaluation and application-integration proposals | Sequence/graph experiments and, if justified, JEPA | No partner participation, dataset access, generalisation result, or model efficiency is implied by this roadmap. See the [claims ledger](docs/reading-list.md#claim-boundaries). diff --git a/com_jepa/forest_demo.py b/com_jepa/forest_demo.py new file mode 100644 index 0000000..956ba9a --- /dev/null +++ b/com_jepa/forest_demo.py @@ -0,0 +1,225 @@ +"""CPU-only Random Forest demonstration on explicitly fictional tabular snapshots. + +Run: python -m com_jepa.forest_demo --output artifacts/forest-demo.json +This is a pipeline exercise, not a validated organisational predictor. +""" + +import argparse +from collections import Counter +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from importlib.metadata import version +import json +import math +from pathlib import Path +import random + +from .events import timestamp + + +FEATURES = ("days_remaining", "unresolved_dependencies", "evidence_age_days", "remaining_work_units") +TARGET = "all_criteria_met_by_version_deadline" +SETTLED = {"met": 1, "not_met": 0} +UNRESOLVED = {"unknown", "right_censored", "disputed"} +GENERATOR_VERSION = "toy-tabular-v1" +START = datetime(2025, 1, 1, tzinfo=timezone.utc) + + +def stamp(value): + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def generate_rows(samples=300, seed=42): + """Invent one snapshot per independent initiative, with a declared noisy rule. + + Numbers and causal-looking relationships are authored assumptions. This does + not consume the event ledger or assign labels to the discussion cards. + """ + if not 120 <= samples <= 2000: + raise ValueError("samples must be between 120 and 2000 for this small demo") + rng = random.Random(seed) + rows = [] + for i in range(samples): + as_of = START + timedelta(days=i) + days, dependencies, age, work = rng.randint(2, 28), rng.randint(0, 4), rng.randint(0, 10), rng.randint(2, 30) + logit = 1.2 + 0.13 * days - 0.75 * dependencies - 0.06 * age - 0.13 * work + probability = 1 / (1 + math.exp(-logit)) + status = "met" if rng.random() < probability else "not_met" + if rng.random() < 0.06: + status = rng.choice(sorted(UNRESOLVED)) + due = as_of + timedelta(days=days) + rows.append({ + "data_origin": "fictional", "generator_version": GENERATOR_VERSION, + "organisation_id": "fictional-org", "initiative_id": f"initiative-{i:04d}", + "commitment_id": f"commitment-{i:04d}", "commitment_version": 1, + "target": TARGET, "as_of": stamp(as_of), "due_at": stamp(due), + "features_available_at": stamp(as_of), + "features": dict(zip(FEATURES, (days, dependencies, age, work))), + "label_status": status, + "label_available_at": stamp(due + timedelta(days=rng.randint(1, 5))), + }) + return rows + + +def feature_matrix(rows): + """Explicit allowlist: neither identities nor future outcome fields are inputs.""" + matrix = [] + for row in rows: + if timestamp(row["features_available_at"]) > timestamp(row["as_of"]): + raise ValueError("features were unavailable at prediction cutoff") + if set(row["features"]) != set(FEATURES): + raise ValueError("unexpected or missing feature; review the allowlist") + values = [row["features"][name] for name in FEATURES] + if any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v) or v < 0 for v in values): + raise ValueError("features must be finite non-negative numbers") + days = (timestamp(row["due_at"]) - timestamp(row["as_of"])).total_seconds() / 86400 + if days <= 0 or days != row["features"]["days_remaining"]: + raise ValueError("days_remaining must match the future version deadline") + matrix.append(values) + return matrix + + +def temporal_split(rows, training_cutoff, test_start, evaluation_as_of): + """One snapshot per initiative; train labels must have settled before fitting. + + This narrow contract deliberately rejects repeated initiatives rather than + pretending to implement full trajectory/component grouping. + """ + train_at, test_at, evaluate_at = map(timestamp, (training_cutoff, test_start, evaluation_as_of)) + if not train_at < test_at <= evaluate_at: + raise ValueError("require training_cutoff < test_start <= evaluation_as_of") + feature_matrix(rows) + seen_initiatives, seen_commitments = set(), set() + train, test, excluded = [], [], Counter() + for row in rows: + if row["data_origin"] != "fictional" or row["generator_version"] != GENERATOR_VERSION: + raise ValueError("this demonstration accepts only its declared fictional row format") + if row["target"] != TARGET or row["commitment_version"] != 1: + raise ValueError("this demo supports only the original version and fixed target") + initiative = row["organisation_id"], row["initiative_id"] + commitment = row["organisation_id"], row["commitment_id"] + if initiative in seen_initiatives or commitment in seen_commitments: + raise ValueError("demo requires one unique initiative and commitment per row") + seen_initiatives.add(initiative) + seen_commitments.add(commitment) + as_of, known = timestamp(row["as_of"]), timestamp(row["label_available_at"]) + if known < timestamp(row["due_at"]): + raise ValueError("assessment availability cannot precede the deadline in this demo") + status = row["label_status"] + if status not in SETTLED and status not in UNRESOLVED: + raise ValueError("unsupported label status") + if as_of < train_at: + if known > train_at: + excluded["training_label_not_yet_available"] += 1 + elif status not in SETTLED: + excluded[f"training_{status}"] += 1 + else: + train.append(row) + elif as_of < test_at: + excluded["temporal_gap"] += 1 + elif as_of > evaluate_at or known > evaluate_at: + excluded["test_not_yet_assessable"] += 1 + elif status not in SETTLED: + excluded[f"test_{status}"] += 1 + else: + test.append(row) + order = lambda r: (r["as_of"], r["initiative_id"]) + return sorted(train, key=order), sorted(test, key=order), dict(sorted(excluded.items())) + + +def run_demo(samples=300, seed=42): + # Optional dependency: the event-validation commands do not import sklearn. + from sklearn.dummy import DummyClassifier + from sklearn.ensemble import RandomForestClassifier + from sklearn.metrics import accuracy_score, brier_score_loss, log_loss + + rows = generate_rows(samples, seed) + training_cutoff = stamp(START + timedelta(days=int(samples * 0.6))) + test_start = stamp(timestamp(training_cutoff) + timedelta(days=30)) + evaluation_as_of = stamp(START + timedelta(days=samples + 40)) + train, test, excluded = temporal_split(rows, training_cutoff, test_start, evaluation_as_of) + x_train, x_test = feature_matrix(train), feature_matrix(test) + y_train = [SETTLED[r["label_status"]] for r in train] + y_test = [SETTLED[r["label_status"]] for r in test] + if len(set(y_train)) != 2 or not y_test: + raise ValueError("need both training classes and at least one settled test outcome") + models = { + "historical_base_rate": DummyClassifier(strategy="prior"), + "random_forest": RandomForestClassifier( + n_estimators=100, max_depth=6, min_samples_leaf=5, random_state=seed, n_jobs=1, + ), + } + scores, probabilities = {}, {} + for name, model in models.items(): + model.fit(x_train, y_train) + positive_column = list(model.classes_).index(1) + predicted = model.predict_proba(x_test)[:, positive_column] + probabilities[name] = predicted.tolist() + scores[name] = { + "brier_score": float(brier_score_loss(y_test, predicted)), + "log_loss": float(log_loss(y_test, predicted, labels=[0, 1])), + "accuracy_at_0_5": float(accuracy_score(y_test, predicted >= 0.5)), + } + payload = json.dumps(rows, sort_keys=True, separators=(",", ":"), allow_nan=False) + return { + "purpose": "synthetic_pipeline_demonstration_not_organisational_evidence", + "generator_version": GENERATOR_VERSION, "seed": seed, + "target": TARGET, "positive_class": "met", + "features": list(FEATURES), "dataset_sha256": sha256(payload.encode()).hexdigest(), + "environment": {p: version(p) for p in ("scikit-learn", "numpy", "scipy")}, + "training_cutoff": training_cutoff, "test_start": test_start, + "evaluation_as_of": evaluation_as_of, + "counts": {"generated": len(rows), "train": len(train), "test": len(test), "excluded": excluded}, + "train_fulfilment_rate": sum(y_train) / len(y_train), + "test_fulfilment_rate": sum(y_test) / len(y_test), + "training_initiatives": [r["initiative_id"] for r in train], + "forest_parameters": models["random_forest"].get_params(), + "metrics": scores, + "test_predictions": [ + {"initiative_id": row["initiative_id"], "as_of": row["as_of"], + "label_available_at": row["label_available_at"], "observed_label": row["label_status"], + "features": row["features"], + **{name: values[i] for name, values in probabilities.items()}} + for i, row in enumerate(test) + ], + "limitations": [ + "Labels follow an invented rule, not observations from organisations.", + "One synthetic organisation; independent initiatives; no revisions, dependencies between rows, or interventions.", + "Same generator in train and test; no real-world transfer or JEPA conclusions.", + "Fixed forest; no tuning, calibration study, uncertainty intervals, or causal claims.", + "Toy snapshots are not exports of the draft event ledger or the fictional challenge cards.", + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--samples", type=int, default=300) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output", type=Path, help="optional JSON report; use ignored artifacts/") + args = parser.parse_args() + try: + report = run_demo(args.samples, args.seed) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2, allow_nan=False) + stream.write("\n") + except ModuleNotFoundError: + parser.exit(1, "Install the optional demo dependencies: python -m pip install -r requirements-ml.txt\n") + except (ValueError, OSError) as error: + parser.exit(1, f"error: {error}\n") + print("SYNTHETIC PIPELINE DEMO — not evidence of organisational prediction quality") + print(f"Rows: {report['counts']['generated']} generated; {report['counts']['train']} train; {report['counts']['test']} test") + print(f"Train labels available by {report['training_cutoff']}; test snapshots from {report['test_start']}") + print(f"Excluded: {json.dumps(report['counts']['excluded'], sort_keys=True)}") + print("Model Brier (lower) Log loss (lower) Accuracy @ 0.5") + for name, metrics in report["metrics"].items(): + print(f"{name:25} {metrics['brier_score']:.4f} {metrics['log_loss']:.4f} {metrics['accuracy_at_0_5']:.4f}") + print("Random Forest probabilities are uncalibrated; no causal or transfer claim.") + if args.output: + print(f"Report: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/com_jepa/tabpfn_demo.py b/com_jepa/tabpfn_demo.py new file mode 100644 index 0000000..e136430 --- /dev/null +++ b/com_jepa/tabpfn_demo.py @@ -0,0 +1,181 @@ +"""Optional TabPFN-3 GPU comparison using the forest demo's exact synthetic split. + +Requires a separately downloaded, hash-verified checkpoint. Does not serve a model. +""" + +import argparse +from datetime import datetime, timezone +from hashlib import file_digest +from importlib.metadata import version +import json +from pathlib import Path +import platform +from time import perf_counter + +from .forest_demo import SETTLED, feature_matrix, generate_rows, run_demo, temporal_split + + +MODEL_REPO = "Prior-Labs/tabpfn_3" +MODEL_REVISION = "24a16a89d245878b846555110985634aa2e656d7" +MODEL_FILENAME = "tabpfn-v3-classifier-v3_default.ckpt" +MODEL_SHA256 = "d0d865d54dfbc524f5703104be90620182dca7e5fb2c16de72e9959ea18f3988" +MODEL_LICENCE = "https://huggingface.co/Prior-Labs/tabpfn_3/blob/" + MODEL_REVISION + "/LICENSE" + + +def verify_checkpoint(path, expected_sha256): + if len(expected_sha256) != 64 or any(c not in "0123456789abcdef" for c in expected_sha256): + raise ValueError("checkpoint SHA-256 must be 64 lowercase hexadecimal characters") + with Path(path).open("rb") as stream: + actual = file_digest(stream, "sha256").hexdigest() + if actual != expected_sha256: + raise ValueError("checkpoint hash mismatch; refusing to load model") + return actual + + +def predict_independently(estimator, rows): + """A future test row must not enter an earlier row's preprocessing/context.""" + import numpy as np + + classes = list(estimator.classes_) + if set(classes) != {0, 1}: + raise ValueError("expected classes 0=not_met and 1=met") + positive_column = classes.index(1) + predictions = [] + for row in rows: + output = np.asarray(estimator.predict_proba(np.asarray([row], dtype=float))) + if output.shape != (1, 2) or not np.isfinite(output).all(): + raise ValueError("invalid probability response") + if (output < 0).any() or (output > 1).any() or not np.allclose(output.sum(axis=1), 1): + raise ValueError("invalid probability distribution") + predictions.append(float(output[0, positive_column])) + return np.asarray(predictions) + + +def comparison(checkpoint, expected_sha256, samples=300, seed=42, gpu_memory_gib=4): + checkpoint = Path(checkpoint).resolve() + if expected_sha256 != MODEL_SHA256: + raise ValueError("this adapter pins one checkpoint; update its provenance before changing weights") + checkpoint_hash = verify_checkpoint(checkpoint, expected_sha256) + if checkpoint.name != MODEL_FILENAME: + raise ValueError("use the explicitly documented TabPFN-3 checkpoint filename") + if not 0 < gpu_memory_gib <= 8: + raise ValueError("GPU allocation budget must be greater than 0 and at most 8 GiB") + + import torch + from sklearn.metrics import accuracy_score, brier_score_loss, log_loss + from tabpfn import TabPFNClassifier + + if not torch.cuda.is_available(): + raise ValueError("this experiment requires a working CUDA GPU; no silent CPU fallback") + properties = torch.cuda.get_device_properties(0) + budget_bytes = int(gpu_memory_gib * 1024**3) + torch.cuda.set_per_process_memory_fraction(min(1.0, budget_bytes / properties.total_memory), 0) + torch.set_num_threads(2) + torch.manual_seed(seed) + torch.cuda.reset_peak_memory_stats(0) + + baseline_start = perf_counter() + report = run_demo(samples, seed) + baseline_seconds = perf_counter() - baseline_start + rows = generate_rows(samples, seed) + train, test, excluded = temporal_split(rows, report["training_cutoff"], report["test_start"], report["evaluation_as_of"]) + if [r["initiative_id"] for r in train] != report["training_initiatives"]: + raise ValueError("training split differs from baseline") + if [r["initiative_id"] for r in test] != [r["initiative_id"] for r in report["test_predictions"]]: + raise ValueError("test split differs from baseline") + if excluded != report["counts"]["excluded"]: + raise ValueError("exclusion policy differs from baseline") + + import numpy as np + + x_train, x_test = feature_matrix(train), feature_matrix(test) + y_train = np.asarray([SETTLED[r["label_status"]] for r in train]) + y_test = np.asarray([SETTLED[r["label_status"]] for r in test]) + estimator = TabPFNClassifier( + model_path=checkpoint, device="cuda", random_state=seed, + n_estimators=4, n_preprocessing_jobs=1, + fit_mode="fit_preprocessors", softmax_temperature=0.9, + ) + torch.cuda.synchronize() + start = perf_counter() + estimator.fit(np.asarray(x_train, dtype=float), y_train) + torch.cuda.synchronize() + fit_seconds = perf_counter() - start + start = perf_counter() + predicted = predict_independently(estimator, x_test) + torch.cuda.synchronize() + predict_seconds = perf_counter() - start + report["metrics"]["tabpfn_3"] = { + "brier_score": float(brier_score_loss(y_test, predicted)), + "log_loss": float(log_loss(y_test, predicted, labels=[0, 1])), + "accuracy_at_0_5": float(accuracy_score(y_test, predicted >= 0.5)), + } + for row, probability in zip(report["test_predictions"], predicted): + row["tabpfn_3"] = float(probability) + report["environment"].update({p: version(p) for p in ("tabpfn", "torch", "huggingface-hub")}) + report["environment"].update({"python": platform.python_version(), "architecture": platform.machine(), + "gpu": properties.name, "cuda_runtime": torch.version.cuda}) + report["candidate"] = { + "name": "TabPFN-3", "repository": MODEL_REPO, "revision": MODEL_REVISION, + "filename": checkpoint.name, "sha256": checkpoint_hash, "licence": MODEL_LICENCE, + "n_estimators": 4, "random_state": seed, "device": "cuda", "n_preprocessing_jobs": 1, + "resolved_n_estimators": int(estimator.n_estimators_), + "fit_mode": "fit_preprocessors", "softmax_temperature": 0.9, + "prediction_protocol": "one_test_row_per_call_with_fixed_training_context", + "gpu_allocator_budget_gib": gpu_memory_gib, + "peak_torch_allocated_bytes": torch.cuda.max_memory_allocated(0), + "peak_torch_reserved_bytes": torch.cuda.max_memory_reserved(0), + } + report["timing_seconds"] = { + "complete_baseline_demo": baseline_seconds, + "tabpfn_fit_including_model_loading": fit_seconds, + "tabpfn_predict_all_test_rows_individually": predict_seconds, + } + report["implementation_sha256"] = {} + report["evaluated_at_utc"] = datetime.now(timezone.utc).isoformat() + for name in ("forest_demo.py", "tabpfn_demo.py"): + with Path(__file__).with_name(name).open("rb") as stream: + report["implementation_sha256"][name] = file_digest(stream, "sha256").hexdigest() + report["limitations"].extend([ + "One pretrained checkpoint, one synthetic generator and seed; no claim of model superiority or organisational validity.", + "TabPFN has external pretraining; this comparison does not isolate architecture from pretraining.", + "GPU timing is from a shared machine and single-row prediction; not a controlled hardware speed benchmark.", + "CUDA seeds do not guarantee bitwise reproducibility across platforms or library versions.", + "Allocator budget is not a hard bound on all process or unified-memory use.", + "TabPFN-3 weights and outputs have their own non-commercial licence; the repository MIT licence does not replace it.", + ]) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--checkpoint-sha256", default=MODEL_SHA256) + parser.add_argument("--samples", type=int, default=300) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--gpu-memory-gib", type=float, default=4) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.output.exists(): + raise ValueError("output already exists; choose a new report filename") + report = comparison(args.checkpoint, args.checkpoint_sha256, args.samples, args.seed, args.gpu_memory_gib) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2, allow_nan=False) + stream.write("\n") + except ModuleNotFoundError as error: + parser.exit(1, f"Missing optional dependency: {error.name}. See docs/tabpfn-comparison.md.\n") + except (ValueError, OSError) as error: + parser.exit(1, f"error: {error}\n") + print("SYNTHETIC GPU COMPARISON — not evidence of organisational prediction quality") + print(f"Train: {report['counts']['train']}; test: {report['counts']['test']}; generator seed: {report['seed']}") + print("Model Brier (lower) Log loss (lower) Accuracy @ 0.5") + for name, metrics in report["metrics"].items(): + print(f"{name:25} {metrics['brier_score']:.4f} {metrics['log_loss']:.4f} {metrics['accuracy_at_0_5']:.4f}") + print(f"TabPFN-3: checkpoint {report['candidate']['sha256']}") + print(f"Report: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/docs/random-forest-demo.md b/docs/random-forest-demo.md new file mode 100644 index 0000000..4b03286 --- /dev/null +++ b/docs/random-forest-demo.md @@ -0,0 +1,82 @@ +# A small Random Forest example + +Status: executable synthetic pipeline demonstration. It trains a real Random Forest on invented tabular rows; it does not train an organisationally validated predictor, JEPA, or a foundation model. + +## Run it locally + +Use Python 3.14 for the pinned ML environment exercised in CI. From the repository root: + +```sh +python3.14 -m venv .venv +.venv/bin/python -m pip install -r requirements-ml.txt +.venv/bin/python -m com_jepa.forest_demo +.venv/bin/python -m com_jepa.forest_demo --output artifacts/forest-demo.json +.venv/bin/python -m unittest discover -s tests -v +``` + +The output option writes a JSON report and refuses to overwrite an existing file; choose a new filename for another experiment. `artifacts/` is ignored by Git. Without that option the model fits in memory, prints a comparison, and exits without saving data or model weights. It needs no GPU, API key, model download, or hosted service. Installing dependencies requires network access once. + +The original `requirements.txt` remains sufficient for the event validator. Tests of the ML fit skip when scikit-learn is absent; the dedicated ML CI job installs it and runs the full suite. + +## What the forest learns + +The question is whether one accepted version meets all its criteria by its deadline. The demo invents 300 rows, each a different commitment in a different initiative within one fictional organisation. One row represents the information available at one prediction cutoff. All commitments are version 1; there are no revisions, shared resources, or interventions in this simplified simulation. + +Four declared features are available at the cutoff: + +| Feature | Invented range | Interpretation for the exercise | +| --- | --- | --- | +| `days_remaining` | 2–28 | Days from the prediction cutoff to the original deadline | +| `unresolved_dependencies` | 0–4 | Count of dependencies believed unresolved at that cutoff | +| `evidence_age_days` | 0–10 | Age of the latest relevant evidence | +| `remaining_work_units` | 2–30 | An invented measure of work remaining, with no cross-organisation meaning | + +These are proposed teaching features, not a ratified extension of the event schema. The generator supplies their values directly. This code is **not** a feature extractor from real event histories. A real adapter must establish the units, provenance, and as-of derivation before reusing a feature name. + +The entire invented probability rule is: + +```text +z = 1.2 + 0.13 × days_remaining + − 0.75 × unresolved_dependencies + − 0.06 × evidence_age_days + − 0.13 × remaining_work_units +p(met) = 1 / (1 + exp(−z)) +``` + +Outcomes are Bernoulli draws from that probability; about 6% are independently marked unknown, right-censored, or disputed. This artificial missingness is much simpler than real observation processes. Labels become available one to five days after the deadline. Coefficients, feature ranges, independence, noise, missingness, and timing are all authored assumptions. They are not organisational findings or causal estimates. + +The forest receives the four features and settled training labels. It does not receive the rule's probability, identities, future observations, or test labels. The same generator produces train and test rows, so a successful result demonstrates learning that artificial relationship. It cannot establish transfer outside the generator. + +## Fit and compare + +[RandomForestClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html) combines predictions from decision trees fitted to resampled training data. Here it uses 100 trees, maximum depth 6, minimum leaf size 5, one CPU worker, and a fixed random seed. The forest's probabilities are estimates from its tree ensemble; they are not automatically calibrated. + +The comparator is [DummyClassifier with `strategy="prior"`](https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyClassifier.html). It predicts the observed training fulfilment rate for every test row. This establishes whether the forest offers anything beyond that simple historical expectation under the toy generator. + +The fit cutoff occurs 60% of the way through the generated daily prediction cutoffs. Only earlier rows whose labels are settled and available by that fit date can train either model. Test cutoffs begin 30 days later. Test labels must be available by the separately declared evaluation date. The report accounts for excluded rows by reason. There is no random train/test shuffle, hyperparameter search, or test-driven model selection. + +There is one row per initiative and commitment. Repeated IDs are rejected, so the toy cannot silently put snapshots of the same initiative on both sides of the split. Real data needs a proper grouped temporal split and dependency-component handling; this restriction is not a general implementation of either. + +## Read the output + +The terminal prints both models' Brier score, log loss, and accuracy at a fixed 0.5 threshold. Lower Brier score and log loss are better; accuracy uses an arbitrary teaching threshold, not an agreed business cost. The JSON report records generation seed/version, dataset hash, split dates, exclusions, training prevalence, model parameters, library versions, and every scored test prediction. + +No test requires the forest to beat the base rate. That would turn the generator's design into a success condition for the research. There are no confidence intervals, calibration experiment, fairness assessment, or causal claims here. Changing the seed demonstrates sampling variation, not robustness across organisations. Keep reported scores labelled **synthetic pipeline results**. + +## Why this does not pre-empt partner discovery + +The generator answers one engineering question: can we fit, evaluate, and inspect a conventional probability model while enforcing the selected temporal boundaries? It does not resolve the ambiguous challenge scenarios, assign labels to them, or enlarge them into a supposed organisational training corpus. + +Partners can still reject these features, their units, the binary target, or the commitment abstraction. Any real dataset should follow the [data contract and stewardship work](data-contract.md) and the [research evaluation plan](research-plan.md), including independent assessment and grouped time splits. Passing this demo does not satisfy those gates. + +## Where tabular foundation models fit + +The structured-data models raised in discussion are useful comparison candidates. Our immediate task is table-based classification; forecasting a regular numerical time series is a different task unless we first define a justified transformation. + +- [Google TabFM](https://research.google/blog/introducing-tabfm-a-zero-shot-foundation-model-for-tabular-data/) describes in-context tabular classification/regression using labelled examples without per-task weight updates. Google reports pretraining on diverse synthetic datasets and evaluation on real tabular benchmarks. That does not validate our much narrower invented organisational rule. +- [Prior Labs' TabPFN-3 documentation](https://docs.priorlabs.ai/changelog/tabpfn-3) describes tabular inference and local weights subject to licence acceptance. It is a possible later comparator; no weights, licence acceptance, API integration, or performance claim are included here. +- [Amazon Chronos-2](https://www.amazon.science/blog/introducing-chronos-2-from-univariate-to-universal-forecasting) concerns time-series forecasting with covariates. Its fit would require a separate time-series target; it is not a drop-in replacement for the classifier demonstrated here. + +For a future comparison, give every model the same eligible historical labels and as-of features. In-context learning still uses task examples; “zero-shot” does not mean we can omit target definition or reveal future labels. Declare whether test rows are processed jointly, and prevent later test-row information from entering earlier forecasts through a shared context. Compare calibration, latency, data movement, licence/access terms, and cost alongside prediction quality. + +PyTorch is not needed for this Random Forest. Add it when an actual neural-model experiment requires it. A tabular foundation model could become a strong baseline for com-jepa, but its availability does not establish that it should always be the default or that JEPA will be better. diff --git a/docs/reading-list.md b/docs/reading-list.md index 21161d3..32facc0 100644 --- a/docs/reading-list.md +++ b/docs/reading-list.md @@ -31,4 +31,4 @@ Further lineage to examine includes language/action approaches to cooperative wo | Shared representations and local adaptation are research questions | One trained model is valid for most organisations | | Durable context can support deliberation by design | The design has been shown to prevent cognitive surrender | | Better forecasts may help people choose interventions | Observational action-conditioned forecasts identify causal effects | -| The schema and fictional replay are executable | A real dataset, trained model, or deployed integration exists here | +| The schema, fictional replay, and synthetic Random Forest demo are executable | A real organisational dataset, validated organisational model, or deployed integration exists here | diff --git a/docs/tabpfn-comparison.md b/docs/tabpfn-comparison.md new file mode 100644 index 0000000..5a7a5f7 --- /dev/null +++ b/docs/tabpfn-comparison.md @@ -0,0 +1,102 @@ +# TabPFN-3 GPU comparison + +This optional adapter compares a pretrained TabPFN-3 classifier with the Random Forest and historical base rate on the **same fictional data, temporal split, feature allowlist, and scoring targets**. It does not change the generator, label the discussion cases, or introduce real organisational data. + +## What is pinned + +| Component | Selection | +| --- | --- | +| TabPFN package | `8.5.0` | +| PyTorch | `2.11.0`, official CUDA 13.0 build for the GPU experiment | +| Model repository | `Prior-Labs/tabpfn_3` | +| Model revision | `24a16a89d245878b846555110985634aa2e656d7` | +| Checkpoint | `tabpfn-v3-classifier-v3_default.ckpt` | +| Checkpoint SHA-256 | `d0d865d54dfbc524f5703104be90620182dca7e5fb2c16de72e9959ea18f3988` | +| Candidate ensemble | Four estimators; automatic ensemble scaling disabled | +| Test protocol | One test row per prediction call, fixed historical training context | + +The adapter verifies the checkpoint hash before loading it. It requires CUDA and does not silently fall back to another model or device. A future checkpoint comparison must update provenance explicitly. + +## Environment and model access + +The GPU setup uses Linux ARM64, Python 3.12, and an NVIDIA GB10. The pinned tabular dependencies also support this Python version. Create an isolated environment on the GPU machine; keep its existing inference service separate. + +```sh +python3.12 -m venv .venv +.venv/bin/python -m pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cu130 +.venv/bin/python -m pip install -r requirements-tabpfn.txt +.venv/bin/python -m pip check +``` + +The [upstream package](https://github.com/PriorLabs/TabPFN) documents local inference. The model weights and outputs have a [separate non-commercial licence](https://huggingface.co/Prior-Labs/tabpfn_3/blob/24a16a89d245878b846555110985634aa2e656d7/LICENSE). The repository's MIT licence does not grant production rights for TabPFN. This example is a non-commercial synthetic evaluation; it does not distribute weights, distil their outputs into another model, or deploy a service. + +After reviewing the applicable model terms, download the fixed revision: + +```sh +.venv/bin/python - <<'PY' +from huggingface_hub import hf_hub_download +from com_jepa.tabpfn_demo import MODEL_REPO, MODEL_REVISION, MODEL_FILENAME +for filename in ('LICENSE', MODEL_FILENAME): + hf_hub_download( + repo_id=MODEL_REPO, revision=MODEL_REVISION, filename=filename, + local_dir='artifacts/tabpfn-3', + ) +PY +``` + +At initial inspection this revision was publicly downloadable without authentication. If access requirements change, use the publisher's authorised access process. The adapter never accepts a licence, logs into an account, or downloads an unspecified replacement model. + +## Run the experiment + +```sh +.venv/bin/python -m unittest discover -s tests -v +.venv/bin/python -m com_jepa validate examples/fictional-trajectory.jsonl +.venv/bin/python -m com_jepa.tabpfn_demo \ + --checkpoint artifacts/tabpfn-3/tabpfn-v3-classifier-v3_default.ckpt \ + --output artifacts/tabpfn-comparison.json +.venv/bin/python -m pip freeze > artifacts/tabpfn-environment.txt +``` + +The output file must be new. The model and reports stay in ignored `artifacts/`. To initiate an already prepared run from another machine, use SSH to the experiment checkout and invoke the same module. Host aliases and paths are operator configuration, not embedded in the adapter. + +The default PyTorch allocator budget is 4 GiB, adjustable with `--gpu-memory-gib` up to 8 GiB. This is not a hard bound on all process or unified-memory use. Check available memory and other workloads before running. The adapter does not stop, replace, or reconfigure an existing model service, and no GPU runner is added to CI. + +## What is compared + +The adapter first runs the original forest demonstration, then reconstructs and verifies identical training/test identities and exclusions. The pretrained candidate gets only the selected training features and labels. It processes each test row separately; it never receives future test rows or any test labels as context. Four feature values and the fixed historical examples are its entire task input. + +TabPFN uses external pretraining, so equal task data does not imply equal total training information or compute. This is a practical estimator comparison, not an experiment isolating model architecture from pretraining. It is also not a JEPA experiment or a test of organisational transfer. + +The JSON report extends the baseline report with candidate metrics, per-row probabilities, checkpoint provenance, code hashes, device/library versions, timing, and peak PyTorch memory. `fit` includes checkpoint loading and preparation of the training context; it is not fine-tuning the neural weights. Prediction time covers individual calls and must not be advertised as maximum batched throughput. The machine may be sharing its GPU with another workload. + +GPU timings and floating-point outputs need not be bitwise reproducible across hardware and versions. Preserve the environment capture alongside the report. A single generator/seed cannot establish broad model superiority, even if one model scores better on this run. Probability calibration, uncertainty intervals, multiple reviewed generators, and real-data evaluation remain separate work. + +## Testing boundaries + +Unit tests verify checkpoint rejection, positive-class mapping, probability validity, and one-row-per-call isolation without downloading a model. The existing temporal and missing-label tests still apply. A real checkpoint execution is a separate, explicitly invoked GPU experiment, not a stubbed test result. + +## First measured run — 8 September 2026 + +**Synthetic pipeline evidence only.** The GPU experiment completed on an NVIDIA GB10 using Python 3.12.3, PyTorch 2.11.0+cu130, and TabPFN 8.5.0. It used generator `toy-tabular-v1`, seed 42, with 300 generated rows, 153 eligible training rows, and 85 eligible test rows. All three estimators received the same eligible task data and were scored against the same outcomes. + +| Model | Brier score ↓ | Log loss ↓ | Accuracy at 0.5 ↑ | +| --- | ---: | ---: | ---: | +| Historical base rate | 0.2198 | 0.6317 | 68.24% | +| Random Forest | 0.1677 | 0.5200 | 78.82% | +| TabPFN-3 | 0.1500 | 0.4760 | 77.65% | + +TabPFN produced better probability scores on this sample. The forest classified 67 of 85 cases correctly at the fixed threshold; TabPFN classified 66. This illustrates why probability scoring matters for decision support: better probability estimates need not produce more correct binary calls at an arbitrary threshold. These scores do not establish calibration or statistical significance. + +The resolved TabPFN ensemble contained four estimators. Loading the checkpoint and preparing the training context took 1.24 seconds; predicting all 85 test rows individually took 6.21 seconds. Peak PyTorch tensor allocation was 228,177,920 bytes (about 218 MiB), which excludes other process and device memory. These measurements come from a shared GPU and are not a controlled speed benchmark. + +Reproduction identifiers: + +- Source commit: [`8053ae348856bdc2e1facaa29501ae881f84ae44`](https://github.com/Reflective-Lab/com-jepa/commit/8053ae348856bdc2e1facaa29501ae881f84ae44). +- Dataset SHA-256: `79064303e37a8da38390cd0635168bad630be1fbc2793d2f427e5b6088670981`. +- Evaluation completed: `2026-09-08T17:09:02.580470+00:00`. +- Checkpoint revision and hash: the pinned values above. +- Local run receipts: `artifacts/tabpfn-comparison-seed42-final.json`, `artifacts/tabpfn-environment.txt`, and `artifacts/tabpfn-source-revision.txt` (ignored by Git). + +All 25 unit tests passed on the GPU machine, and the real checkpoint completed the comparison separately. The existing inference service returned a successful health response after the experiment. + +This establishes that the comparison can run on modest task data without fine-tuning neural weights. It does not establish organisational prediction quality, transfer across organisations, or a benefit from JEPA. The next research step is to challenge the data and evaluation assumptions with partners before treating this ranking as a model-selection result. diff --git a/requirements-ml.txt b/requirements-ml.txt new file mode 100644 index 0000000..a8f3678 --- /dev/null +++ b/requirements-ml.txt @@ -0,0 +1,9 @@ +# Optional CPU-only demo, tested with Python 3.14. Base validation stays separate. +-r requirements.txt +scikit-learn==1.9.0 +numpy==2.5.3 +scipy==1.18.1 +joblib==1.6.0 +cloudpickle==3.1.2 +narwhals==2.26.0 +threadpoolctl==3.6.0 diff --git a/requirements-tabpfn.txt b/requirements-tabpfn.txt new file mode 100644 index 0000000..f7804c3 --- /dev/null +++ b/requirements-tabpfn.txt @@ -0,0 +1,4 @@ +# Install CUDA PyTorch from its official index first; see the comparison guide. +-r requirements-ml.txt +torch==2.11.0 +tabpfn==8.5.0 diff --git a/tests/test_forest_demo.py b/tests/test_forest_demo.py new file mode 100644 index 0000000..ea57f61 --- /dev/null +++ b/tests/test_forest_demo.py @@ -0,0 +1,89 @@ +from copy import deepcopy +from importlib.util import find_spec +import math +import unittest + +from com_jepa.forest_demo import FEATURES, feature_matrix, generate_rows, run_demo, temporal_split + + +class ForestDataBoundaryTests(unittest.TestCase): + def setUp(self): + self.rows = generate_rows() + self.cutoffs = ("2025-06-30T00:00:00Z", "2025-07-30T00:00:00Z", "2025-12-07T00:00:00Z") + + def test_future_labels_are_not_training_examples(self): + self.rows[0]["label_available_at"] = "2025-07-01T00:00:00Z" + train, test, excluded = temporal_split(self.rows, *self.cutoffs) + self.assertNotIn(self.rows[0], train) + self.assertGreater(excluded["training_label_not_yet_available"], 0) + self.assertTrue(all(r["label_available_at"] <= self.cutoffs[0] for r in train)) + self.assertTrue(all(r["as_of"] >= self.cutoffs[1] for r in test)) + self.assertFalse({r["initiative_id"] for r in train} & {r["initiative_id"] for r in test}) + self.assertEqual(len(train) + len(test) + sum(excluded.values()), len(self.rows)) + + def test_unknown_censored_disputed_are_not_failures(self): + for status in ("unknown", "right_censored", "disputed"): + with self.subTest(status=status): + rows = deepcopy(self.rows) + rows[0]["label_status"] = status + train, _, excluded = temporal_split(rows, *self.cutoffs) + self.assertNotIn(rows[0], train) + self.assertGreater(excluded[f"training_{status}"], 0) + + def test_future_features_and_feature_label_leakage_are_rejected(self): + row = deepcopy(self.rows[0]) + row["features_available_at"] = "2025-01-02T00:00:00Z" + with self.assertRaisesRegex(ValueError, "unavailable"): + feature_matrix([row]) + row = deepcopy(self.rows[0]) + row["features"]["label_status"] = 1 + with self.assertRaisesRegex(ValueError, "allowlist"): + feature_matrix([row]) + + def test_outcomes_and_identifiers_do_not_change_feature_matrix(self): + altered = deepcopy(self.rows) + for row in altered: + row["label_status"] = "unknown" + row["initiative_id"] = "ignored-by-feature-builder" + self.assertEqual(feature_matrix(self.rows), feature_matrix(altered)) + self.assertEqual(len(feature_matrix(self.rows)[0]), len(FEATURES)) + + def test_repeated_initiatives_cannot_cross_the_split(self): + self.rows[-1]["initiative_id"] = self.rows[0]["initiative_id"] + with self.assertRaisesRegex(ValueError, "unique initiative"): + temporal_split(self.rows, *self.cutoffs) + + def test_test_labels_after_evaluation_cutoff_are_excluded(self): + self.rows[-1]["label_available_at"] = "2026-01-01T00:00:00Z" + _, test, excluded = temporal_split(self.rows, *self.cutoffs) + self.assertNotIn(self.rows[-1], test) + self.assertGreater(excluded["test_not_yet_assessable"], 0) + + def test_nonfinite_features_and_inconsistent_deadlines_are_rejected(self): + for value in (math.nan, math.inf, -1): + with self.subTest(value=value): + row = deepcopy(self.rows[0]) + row["features"]["evidence_age_days"] = value + with self.assertRaises(ValueError): + feature_matrix([row]) + self.rows[0]["features"]["days_remaining"] += 1 + with self.assertRaisesRegex(ValueError, "deadline"): + feature_matrix(self.rows) + + +@unittest.skipUnless(find_spec("sklearn"), "optional ML demo: install requirements-ml.txt") +class ForestExecutionTests(unittest.TestCase): + def test_reproducible_finite_predictions_and_training_only_base_rate(self): + first, second = run_demo(), run_demo() + self.assertEqual(first, second) + self.assertIn("synthetic_pipeline", first["purpose"]) + for row in first["test_predictions"]: + self.assertAlmostEqual(row["historical_base_rate"], first["train_fulfilment_rate"]) + self.assertTrue(0 <= row["random_forest"] <= 1) + for scores in first["metrics"].values(): + self.assertTrue(all(math.isfinite(v) for v in scores.values())) + # No assertion that the forest must win: that would canonise the generator. + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tabpfn_demo.py b/tests/test_tabpfn_demo.py new file mode 100644 index 0000000..0b63427 --- /dev/null +++ b/tests/test_tabpfn_demo.py @@ -0,0 +1,53 @@ +from hashlib import sha256 +from importlib.util import find_spec +from pathlib import Path +import tempfile +import unittest + +from com_jepa.tabpfn_demo import predict_independently, verify_checkpoint + + +class CheckpointTests(unittest.TestCase): + def test_changed_checkpoint_is_rejected_before_loading(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "checkpoint.ckpt" + path.write_bytes(b"fictional checkpoint for hash test") + expected = sha256(path.read_bytes()).hexdigest() + self.assertEqual(verify_checkpoint(path, expected), expected) + path.write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + verify_checkpoint(path, expected) + with self.assertRaisesRegex(ValueError, "64 lowercase"): + verify_checkpoint(path, "invalid") + + +@unittest.skipUnless(find_spec("numpy"), "optional ML adapter tests: install requirements-ml.txt") +class PredictionIsolationTests(unittest.TestCase): + def test_each_call_sees_one_row_and_positive_class_is_looked_up(self): + class Estimator: + classes_ = [1, 0] + + def __init__(self): + self.seen = [] + + def predict_proba(self, rows): + self.seen.append(rows.tolist()) + return [[float(rows[0][0]), 1 - float(rows[0][0])]] + + estimator = Estimator() + self.assertEqual(predict_independently(estimator, [[0.2], [0.8]]).tolist(), [0.2, 0.8]) + self.assertEqual(estimator.seen, [[[0.2]], [[0.8]]]) + + def test_invalid_probabilities_fail_instead_of_becoming_metrics(self): + class Estimator: + classes_ = [0, 1] + + def predict_proba(self, rows): + return [[0.4, 0.8]] + + with self.assertRaisesRegex(ValueError, "distribution"): + predict_independently(Estimator(), [[0.2]]) + + +if __name__ == "__main__": + unittest.main()