diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 69ce1de9..62f36366 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -153,7 +153,7 @@ from darnit_baseline.controls import level1 # CORRECT — use plugin discovery from darnit.core.discovery import get_implementation impl = get_implementation("openssf-baseline") -controls = impl.get_all_controls() +config_path = impl.get_framework_config_path() ``` --- @@ -380,13 +380,8 @@ class ComplianceImplementation(Protocol): @property def spec_version(self) -> str: ... # "OSPS v2025.10.10" - def get_all_controls(self) -> list[ControlSpec]: ... - def get_controls_by_level(self, level: int) -> list[ControlSpec]: ... - def get_rules_catalog(self) -> dict[str, Any]: ... - def get_remediation_registry(self) -> dict[str, Any]: ... def get_framework_config_path(self) -> Path | None: ... - def register_controls(self) -> None: ... - # Optional: register_handlers() — checked via hasattr() + # Optional: register_handlers() — the handler hook, checked via hasattr() ``` ### Entry Point Registration diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d2864ce..176c1131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`classify_writeback`), the threat-model classes in `darnit_baseline.threat_model.models` other than `StrideCategory`, and `darnit.remediation.RemediationResult.to_markdown` (#487). +- **BREAKING:** `ComplianceImplementation` no longer declares + `get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, + `get_remediation_registry`, or `register_controls`, and the framework no + longer calls `register_controls()`. No production path called the others. + The protocol is `name`, `display_name`, `version`, `spec_version`, and + `get_framework_config_path()`; controls, SARIF rules, and remediations come + from the framework TOML (`load_framework_by_name`, + `load_controls_from_framework`). A plugin that still defines the methods is + discovered as before. The in-tree implementations drop them, and + `darnit-example` drops its `_RULES` catalog, `remediation/registry.py`, and + empty `controls` package (#487). ### Added @@ -323,6 +334,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **BREAKING:** `register_handlers()` is the one handler registration hook + for plugins (#451). `darnit-gittuf` and `darnit-reproducibility` rename + `register_sieve_handlers()` to `register_handlers()`, and their `register()` + entry points no longer register handlers during discovery; + `darnit-example` defines only `register_handlers()`; `darnit-csl` no longer + registers `csl_llm_if_present` when `darnit_csl` is imported (call + `CommunitySpecImplementation().register_handlers()`, which the framework + does on every audit and when a framework load needs the step type); `darnit-hello` defines it as a + no-op. The framework still calls `register_sieve_handlers()` on + out-of-tree plugins, for compatibility only. - **BREAKING:** framework loading is strict. An unknown control key, a step key that is neither a common step field nor a setting its step type declares (a misspelled `fail_on_mis`, for example), an `expr` its step diff --git a/CLAUDE.md b/CLAUDE.md index 04d0dfa1..5db66a5a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,7 +55,7 @@ from darnit_baseline.controls import level1 from darnit.core.discovery import get_implementation impl = get_implementation("openssf-baseline") if impl: - controls = impl.get_all_controls() + config_path = impl.get_framework_config_path() ``` ### Rule 2: Implementations MAY Import Framework @@ -78,14 +78,12 @@ impl.name # str: Implementation identifier impl.display_name # str: Human-readable name impl.version # str: Implementation version impl.spec_version # str: Spec version implemented -impl.get_all_controls() # List[ControlSpec]: All controls -impl.get_controls_by_level(n) # List[ControlSpec]: Controls at level n -impl.get_rules_catalog() # Dict: SARIF rule definitions -impl.get_remediation_registry() # Dict: Auto-fix mappings impl.get_framework_config_path() # Path | None: TOML config location -impl.register_controls() # None: Register TOML controls +impl.register_handlers() # None: optional hook, registers Python handlers ``` +Those five members are the whole protocol; `register_handlers()` is optional. Controls, SARIF rules, and remediations come from the framework TOML, never from the implementation. `get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, and `register_controls` were removed (#487); a plugin that still defines them is discovered, and they are not called. + ## Plugin System ### Entry Points @@ -103,8 +101,8 @@ openssf-baseline = "darnit_baseline:register" ```python # my_framework/implementation.py +from importlib.resources import files from pathlib import Path -from darnit.core.plugin import ComplianceImplementation, ControlSpec class MyFrameworkImplementation: @property @@ -123,15 +121,11 @@ class MyFrameworkImplementation: def spec_version(self) -> str: return "MySpec v1.0" - def get_all_controls(self) -> list[ControlSpec]: - # Return your control definitions - ... - def get_framework_config_path(self) -> Path | None: - return Path(__file__).parent / "my-framework.toml" + return Path(str(files(__package__) / "my-framework.toml")) - def register_controls(self) -> None: - pass # Controls are defined in TOML; no Python registration needed + def register_handlers(self) -> None: + pass # Optional: register custom step types (see Handler Registration) ``` 2. Add the registration function: @@ -318,7 +312,9 @@ Remediation plans, then applies only what it planned (feature 043; framework-des ### Handler Registration -Plugins register handlers using the `register_handlers()` method: +`register_handlers()` is the protocol hook for a plugin's Python handlers: sieve step types (`get_sieve_handler_registry().register(...)`, with ceiling and settings, framework-design.md 3.0.3) and MCP tool handlers. The framework calls it on every audit, on `darnit list`, and when loading a framework names a step type that is not yet registered, so it must be safe to call repeatedly. `register_sieve_handlers()` is still called for compatibility with out-of-tree plugins, but is not a supported choice; no in-tree plugin uses it. Registering at module import works but is not the protocol: the framework cannot introspect it, and a registry reset loses the handlers (#451; framework-design.md 6.4). + +MCP tool handlers, for example: ```python class MyImplementation: @@ -373,10 +369,10 @@ Always handle missing implementations gracefully: ```python impl = get_implementation("openssf-baseline") if impl: - result = impl.get_all_controls() + config_path = impl.get_framework_config_path() else: logger.warning("No implementation found") - result = [] + config_path = None ``` ## Technology Stack diff --git a/docs/HANDLER_AUTHORING.md b/docs/HANDLER_AUTHORING.md index 82cf3b06..c82fa246 100644 --- a/docs/HANDLER_AUTHORING.md +++ b/docs/HANDLER_AUTHORING.md @@ -286,10 +286,11 @@ Guidelines for custom handlers: ## 5. Registering a Custom Handler Custom sieve handlers are registered with the `SieveHandlerRegistry` in your -implementation's `register_sieve_handlers()` method: +implementation's `register_handlers()` method, the protocol hook the framework +calls before it loads or audits your framework (framework-design.md 6.4): ```python -def register_sieve_handlers(self) -> None: +def register_handlers(self) -> None: from darnit.sieve.handler_registry import get_sieve_handler_registry from . import handlers @@ -325,10 +326,13 @@ steps = [ ] ``` -Important distinction: - -- `register_sieve_handlers()` is for `[[controls."...".passes]]` handlers. -- `register_handlers()` is for MCP tool handlers under `[mcp.tools.*]`. +`register_handlers()` registers both kinds of handler: sieve step types for +`[[controls."...".passes]]` (in `get_sieve_handler_registry()`) and MCP tool +handlers for `[mcp.tools.*]` (in `darnit.core.handlers.get_handler_registry()`). +A method named `register_sieve_handlers()` is still called for compatibility +with older plugins, but new plugins should not use it. Registering at module +import works, but the framework cannot see it, and a registry reset loses the +handlers. ## 6. Testing One Control at a Time @@ -396,7 +400,7 @@ steps = ["Confirm the README body explains the project"] 2. Add the handler function to `src//handlers.py`. -3. Register it in `register_sieve_handlers()`. +3. Register it in `register_handlers()`. 4. Test the handler directly with pytest. diff --git a/docs/IMPLEMENTATION_GUIDE.md b/docs/IMPLEMENTATION_GUIDE.md index 94e531b6..261824e6 100644 --- a/docs/IMPLEMENTATION_GUIDE.md +++ b/docs/IMPLEMENTATION_GUIDE.md @@ -280,12 +280,7 @@ class ComplianceImplementation(Protocol): @property def spec_version(self) -> str: ... - def get_all_controls(self) -> list[ControlSpec]: ... - def get_controls_by_level(self, level: int) -> list[ControlSpec]: ... - def get_rules_catalog(self) -> dict[str, Any]: ... - def get_remediation_registry(self) -> dict[str, Any]: ... def get_framework_config_path(self) -> Path | None: ... - def register_controls(self) -> None: ... ``` ### Required properties (4) @@ -297,31 +292,35 @@ class ComplianceImplementation(Protocol): | `version` | `str` | Implementation version (e.g., `"0.1.0"`) | | `spec_version` | `str` | Spec version implemented (e.g., `"MySpec v1.0"`) | -### Required methods (6) +### Required method (1) | Method | Purpose | |--------|---------| -| `get_all_controls()` | Return all controls as `ControlSpec` objects | -| `get_controls_by_level(level)` | Filter controls by maturity level | -| `get_rules_catalog()` | Return SARIF rule definitions for output formatting | -| `get_remediation_registry()` | Return mapping of remediation categories to fix functions | | `get_framework_config_path()` | Return path to the TOML configuration file | -| `register_controls()` | Import Python control modules to trigger registration | + +Controls, SARIF rule metadata, and remediations all come from that TOML file; +the implementation class does not return them. `get_all_controls`, +`get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, and +`register_controls` were removed from the protocol in 0.2.0 (#487). A plugin +that still defines them is discovered as before, and they are not called. ### Optional methods | Method | Purpose | |--------|---------| -| `register_handlers()` | Register MCP tool handlers (checked via `hasattr`) | +| `register_handlers()` | The handler hook: register custom sieve step types and MCP tool handlers (checked via `hasattr`; see Section 5) | + +`register_handlers()` is the one supported name. The framework also calls a +`register_sieve_handlers()` method, for compatibility with plugins written +before 0.2.0 only (#451). Registering handlers as a side effect of importing +your module works, but the framework cannot see it, and a registry reset loses +the handlers. ### Minimal implementation ```python # src/darnit_mystandard/implementation.py from pathlib import Path -from typing import Any - -from darnit.core.plugin import ControlSpec class MyStandardImplementation: @@ -343,36 +342,12 @@ class MyStandardImplementation: def spec_version(self) -> str: return "MySpec v1.0" - def get_all_controls(self) -> list[ControlSpec]: - controls = [] - for level in [1, 2, 3]: - controls.extend(self.get_controls_by_level(level)) - return controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - # For now, delegate to the sieve registry - from darnit.sieve.registry import get_control_registry - registry = get_control_registry() - return registry.get_specs_by_level(level) - - def get_rules_catalog(self) -> dict[str, Any]: - return {} # Populate as needed for SARIF output - - def get_remediation_registry(self) -> dict[str, Any]: - from .remediation.registry import REMEDIATION_REGISTRY - return REMEDIATION_REGISTRY - def get_framework_config_path(self) -> Path | None: # Navigate from implementation.py to the TOML file return Path(__file__).parent.parent.parent / "mystandard.toml" - def register_controls(self) -> None: - """TOML-first: controls are defined in the TOML config file. - - No Python registration is needed. Override this only if you need - to register custom sieve handlers (see Section 5). - """ - pass + def register_handlers(self) -> None: + """Optional. Register custom sieve step types here (see Section 5).""" ``` The `get_framework_config_path()` method deserves attention: it must return the @@ -1021,7 +996,7 @@ framework knows which plugin owns each handler: ```python from darnit.sieve.handler_registry import get_sieve_handler_registry -def register_sieve_handlers(self): +def register_handlers(self): registry = get_sieve_handler_registry() registry.set_plugin_context(self.name) @@ -1332,10 +1307,12 @@ def api_check_handler(config: dict, context: HandlerContext) -> HandlerResult: ### Registering the handler -Register handlers in your implementation's `register_sieve_handlers()` method: +Register handlers in your implementation's `register_handlers()` method, the +protocol hook the framework calls before it loads or audits your framework +(framework-design.md 6.4): ```python -def register_sieve_handlers(self) -> None: +def register_handlers(self) -> None: from darnit.sieve.handler_registry import get_sieve_handler_registry from . import handlers @@ -1369,99 +1346,27 @@ steps = [ ## 7. Remediation -Remediation maps audit failures to automated fix actions. The registry tells the -framework which function to call when a control fails. - -### Registry structure - -```python -# src/darnit_mystandard/remediation/registry.py -from typing import Any - -REMEDIATION_REGISTRY: dict[str, dict[str, Any]] = { - "security_policy": { - "description": "Create SECURITY.md with vulnerability reporting info", - "controls": ["MS-SEC-01", "MS-SEC-02"], - "function": "create_security_policy", - "safe": True, # Safe to auto-apply without confirmation - "requires_api": False, # Doesn't need GitHub API access - }, -} -``` - -### Registry fields - -| Field | Type | Purpose | -|-------|------|---------| -| `description` | `str` | Human-readable description of the fix | -| `controls` | `list[str]` | Control IDs this remediation addresses | -| `function` | `str` | Name of the function to call | -| `safe` | `bool` | Whether auto-application is safe | -| `requires_api` | `bool` | Whether the fix needs API access | -| `requires_context` | `list[dict]` | Context values needed before applying | +Remediation is declared in the framework TOML, under each control's +`remediation` table (Section 3, "Remediation in TOML"; framework-design.md +section 4). The framework's remediation executor reads it from there; nothing +in your implementation class returns it. The Python `REMEDIATION_REGISTRY` +dict that earlier versions of this guide described was read only by +`get_remediation_registry()`, which was removed in 0.2.0 (#487). ### Context requirements -Some remediations need user-confirmed context before they can run. A requirement -is met by the key's standing (framework-design.md 7.3, 7.4): a candidate or unknown -key is never ready. Independently of `requires_context`, a template or `when` -clause that reads a key without a usable value stops the control with -`confirmation required: ` and writes nothing (framework-design.md 7.11): - -```python -"codeowners": { - "description": "Create CODEOWNERS file", - "controls": ["MS-GV-01"], - "function": "create_codeowners", - "safe": True, - "requires_api": False, - "requires_context": [{ - "key": "maintainers", - "required": True, - "confidence_threshold": 0.9, - "prompt_if_auto_detected": True, - "warning": "Please confirm who should be code owners.", - }], -}, -``` - -### Remediation action functions - -The actual remediation functions are defined separately and invoked by name from the -registry. They typically create or modify files in the repository: - -```python -# src/darnit_mystandard/remediation/actions.py +Some remediations need user-confirmed context before they can run. Declare them +with `[[controls."ID".remediation.requires_context]]` (framework-design.md 7.3). +A requirement is met by the key's standing (framework-design.md 7.3, 7.4): a +candidate or unknown key is never ready. Independently of `requires_context`, a +template or `when` clause that reads a key without a usable value stops the +control with `confirmation required: ` and writes nothing +(framework-design.md 7.11). -def create_security_policy(owner: str, repo: str, local_path: str, **kwargs) -> dict: - """Create a SECURITY.md file. - - Returns: - dict with keys: success (bool), message (str), files_created (list) - """ - import os - - security_path = os.path.join(local_path, "SECURITY.md") - if os.path.exists(security_path): - return { - "success": True, - "message": "SECURITY.md already exists", - "files_created": [], - } - - content = f"# Security Policy\n\nReport vulnerabilities to security@{owner}.example.com\n" - with open(security_path, "w") as f: - f.write(content) - - return { - "success": True, - "message": "Created SECURITY.md", - "files_created": ["SECURITY.md"], - } -``` - -> **Reference**: See `packages/darnit-baseline/src/darnit_baseline/remediation/registry.py` -> for the full OpenSSF Baseline remediation registry with 11 categories. +A remediation that needs logic the built-in remediation handlers cannot express +is a plugin remediation handler, registered in `register_handlers()` like a +sieve step type (Section 5). It returns planned `FileChange`s and never writes +itself (framework-design.md 4.2). --- @@ -1690,14 +1595,14 @@ def test_full_audit(tmp_path): (tmp_path / "LICENSE").write_text("MIT License") (tmp_path / "SECURITY.md").write_text("Report to security@test.com") + from darnit.config import load_controls_from_framework + from darnit.config.merger import load_framework_config from darnit_mystandard.implementation import MyStandardImplementation impl = MyStandardImplementation() - # Register controls - impl.register_controls() - - # Get controls and verify they loaded - controls = impl.get_all_controls() + # Load the controls from the framework TOML and verify they loaded + config = load_framework_config(impl.get_framework_config_path()) + controls = load_controls_from_framework(config) assert len(controls) > 0 ``` @@ -1810,9 +1715,7 @@ unexpected results since the orchestrator assumes the order. ``` Properties: name, display_name, version, spec_version -Methods: get_all_controls(), get_controls_by_level(level), - get_rules_catalog(), get_remediation_registry(), - get_framework_config_path(), register_controls() +Methods: get_framework_config_path() Optional: register_handlers() ``` @@ -1864,7 +1767,6 @@ from darnit.core.handlers import get_handler_registry | MCP tool handler registry | `packages/darnit/src/darnit/core/handlers.py` | | Reference implementation | `packages/darnit-baseline/src/darnit_baseline/implementation.py` | | Reference TOML | `packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml` | -| Reference remediation | `packages/darnit-baseline/src/darnit_baseline/remediation/registry.py` | | Example implementation | `packages/darnit-example/src/darnit_example/implementation.py` | | Example TOML config | `packages/darnit-example/example-hygiene.toml` | | Example custom handlers | `packages/darnit-example/src/darnit_example/handlers.py` | diff --git a/docs/architecture/example-plugin.md b/docs/architecture/example-plugin.md index 06c5f6d7..28013cf1 100644 --- a/docs/architecture/example-plugin.md +++ b/docs/architecture/example-plugin.md @@ -5,25 +5,25 @@ The `darnit-example` package SHALL export a `register()` function that returns a #### Scenario: Protocol compliance check - **WHEN** `register()` is called -- **THEN** the returned object satisfies all required protocol properties (`name`, `display_name`, `version`, `spec_version`) and methods (`get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, `get_framework_config_path`, `register_controls`) +- **THEN** the returned object satisfies all required protocol properties (`name`, `display_name`, `version`, `spec_version`) and the method `get_framework_config_path` #### Scenario: Entry point discovery - **WHEN** the package is installed via `uv sync` - **THEN** darnit's plugin discovery system finds it under the `darnit.implementations` entry point group with key `example-hygiene` ### Requirement: Package defines 8 controls across 2 levels -The implementation SHALL define exactly 8 controls: 6 at level 1 and 2 at level 2. Control IDs SHALL follow the `PH-{DOMAIN}-NN` format. +The framework TOML SHALL define exactly 8 controls: 6 at level 1 and 2 at level 2. Control IDs SHALL follow the `PH-{DOMAIN}-NN` format. #### Scenario: Level 1 controls -- **WHEN** `get_controls_by_level(1)` is called -- **THEN** 6 controls are returned with IDs `PH-DOC-01`, `PH-DOC-02`, `PH-DOC-03`, `PH-SEC-01`, `PH-CFG-01`, `PH-CFG-02` +- **WHEN** the controls are loaded from `get_framework_config_path()` with `load_controls_from_framework` +- **THEN** 6 controls have level 1, with IDs `PH-DOC-01`, `PH-DOC-02`, `PH-DOC-03`, `PH-SEC-01`, `PH-CFG-01`, `PH-CFG-02` #### Scenario: Level 2 controls -- **WHEN** `get_controls_by_level(2)` is called -- **THEN** 2 controls are returned with IDs `PH-QA-01`, `PH-CI-01` +- **WHEN** the controls are loaded from `get_framework_config_path()` with `load_controls_from_framework` +- **THEN** 2 controls have level 2, with IDs `PH-QA-01`, `PH-CI-01` #### Scenario: Total control count -- **WHEN** `get_all_controls()` is called +- **WHEN** the controls are loaded from `get_framework_config_path()` with `load_controls_from_framework` - **THEN** exactly 8 `ControlSpec` instances are returned ### Requirement: TOML-defined controls use file_exists pass @@ -83,7 +83,7 @@ The package SHALL provide remediation actions `create_readme` and `create_gitign - **THEN** the existing file is not modified and the result status is `"skipped"` ### Requirement: Handler registration with plugin context -The implementation SHALL provide a `register_handlers()` method that registers at least one handler with the framework's handler registry. The handler SHALL be tagged with the plugin name `"example-hygiene"`. +The implementation SHALL provide a `register_handlers()` method, its only handler registration method, that registers its sieve step types and at least one MCP tool handler with the framework's handler registries. The handler SHALL be tagged with the plugin name `"example-hygiene"`. #### Scenario: Handler appears in registry - **WHEN** `register_handlers()` is called diff --git a/docs/architecture/framework-design.md b/docs/architecture/framework-design.md index 7821cfee..e09db85f 100644 --- a/docs/architecture/framework-design.md +++ b/docs/architecture/framework-design.md @@ -1,6 +1,6 @@ # Darnit Framework Design Specification -> **Version**: 1.0.0-alpha.15 +> **Version**: 1.0.0-alpha.16 > **Status**: Authoritative > **Last Updated**: 2026-10-08 @@ -1358,7 +1358,7 @@ openssf-baseline = "darnit_baseline:register" ### 6.3 Implementation Protocol ```python -from darnit.core.plugin import ComplianceImplementation, ControlSpec +from pathlib import Path class MyImplementation: @property @@ -1377,30 +1377,27 @@ class MyImplementation: def spec_version(self) -> str: return "MySpec v1.0" - def get_all_controls(self) -> list[ControlSpec]: - # Return control definitions - ... - def get_framework_config_path(self) -> Path | None: - # Return path to TOML config - return Path(__file__).parent / "my-framework.toml" - - def register_controls(self) -> None: - # No-op. Control definitions MUST come from TOML. - # This method exists for protocol compatibility only. - pass + # Path to the framework TOML, the source of every control definition + ... def register_handlers(self) -> None: - # Register custom sieve/remediation handlers + # Optional: register custom step types and MCP tool handlers (section 6.4) ... ``` -The `register_controls()` method SHALL be a no-op. Implementations MUST NOT use this method to register `ControlSpec` objects with `passes` fields populated. All control definitions MUST originate from TOML configuration files and be loaded via the framework's TOML control loader. The only supported extension point for custom checking logic is `register_handlers()`, which registers named handler functions callable from TOML pass definitions. +`ComplianceImplementation` (`darnit.core.plugin`) is a runtime-checkable protocol with exactly five required members: the properties `name`, `display_name`, `version`, and `spec_version`, and the method `get_framework_config_path()`. Discovery (`darnit.implementations` entry points) accepts an object that has all five and rejects one that lacks any. Extra attributes do not affect discovery, so a plugin that still defines a removed method (Appendix C) is discovered as before; the framework never calls it. + +All control definitions MUST originate from the framework TOML and be loaded by the framework's TOML control loader. The protocol has no method that returns or registers controls, SARIF rules, or remediations; the framework reads all three from the TOML. The only supported extension point for custom checking logic is `register_handlers()`, which registers named step types callable from TOML pass definitions. + +#### Scenario: Plugin still defines a removed protocol method +- **WHEN** a discovered implementation defines `get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, or `register_controls` in addition to the five required members +- **THEN** discovery MUST accept it +- **AND** no audit, listing, or remediation path SHALL call those methods -#### Scenario: Implementation calls register_controls -- **WHEN** the audit pipeline calls `impl.register_controls()` -- **THEN** no `ControlSpec` objects SHALL be registered in the global registry -- **AND** no side-effect imports of control definition modules SHALL occur +#### Scenario: Plugin lacks a required member +- **WHEN** an object returned by a `darnit.implementations` entry point has no `get_framework_config_path` +- **THEN** discovery MUST NOT register it as an implementation #### Scenario: Plugin extends checking with custom handler - **WHEN** a plugin needs custom checking logic beyond built-in pass types @@ -1430,7 +1427,20 @@ The global registry is process-wide and keyed by control id, so it holds every c ### 6.4 Handler Registration -Implementations can register handlers by short name for TOML reference: +`register_handlers()` is the protocol hook through which an implementation registers its Python handlers: sieve step types and remediation handlers in `SieveHandlerRegistry` (section 3.0.3, section 12), and MCP tool handlers in the tool handler registry. It is optional, not one of the required members (section 6.3), so an implementation with no Python handlers may omit it; the reference template (`darnit-hello`) defines it as a documented no-op to show the shape. The framework calls it through `register_implementation_handlers` at the start of every audit and of `darnit list`, and while loading a framework whose steps name a step type not yet registered, whether or not plugin discovery has already run in the process. It MUST be safe to call more than once. + +A method named `register_sieve_handlers()` is called at the same points, as well as `register_handlers()` when both exist, for compatibility with plugins written before the hook was standardized (#451). It is not a supported choice for new plugins, and every in-tree plugin uses `register_handlers()`. Registering handlers as a side effect of importing the plugin's module also works, but the framework cannot see it: it cannot tell whether such a plugin has handlers, and a registry reset is not followed by a re-registration. + +#### Scenario: Plugin registers through the hook +- **WHEN** an audit, `darnit list`, or a framework load needs an implementation's step types +- **THEN** the framework SHALL call that implementation's `register_handlers()` +- **AND** the step types it registers SHALL be available whether or not plugin discovery had already run in the process + +#### Scenario: Plugin written against the old name +- **WHEN** an implementation defines only `register_sieve_handlers()` +- **THEN** the framework SHALL call it at each of the points above, and its step types SHALL be available to the audit + +MCP tool handlers are registered by short name for TOML reference: ```python def register_handlers(self) -> None: @@ -1926,7 +1936,7 @@ Project context from `.project/` SHALL be used to inform WHERE the sieve looks f ## 12. Handler Registry -The framework SHALL provide a handler registry where handlers are registered by name with a phase affinity. Core SHALL register built-in handlers: `file_exists`, `exec`, `gh_api`, `regex`, `pattern`, `llm_eval`, `llm_extract`, `manual`, `manual_steps`, `mcp`, `file_create`, `platform_setting`, `project_update`, `yaml_inject`. Each registration declares its ceiling (section 3.0.1), its `settings` and `expression_names` (section 3.0.3), and, for remediation handlers, plan support (`supports_plan`, section 4.2). Core registers before any plugin, and the registry refuses a plugin registration that would replace a core step type or another plugin's step type (section 3.0.3). Implementations SHALL register domain-specific handlers via the existing `ComplianceImplementation.register_handlers()` method. +The framework SHALL provide a handler registry where handlers are registered by name with a phase affinity. Core SHALL register built-in handlers: `file_exists`, `exec`, `gh_api`, `regex`, `pattern`, `llm_eval`, `llm_extract`, `manual`, `manual_steps`, `mcp`, `file_create`, `platform_setting`, `project_update`, `yaml_inject`. Each registration declares its ceiling (section 3.0.1), its `settings` and `expression_names` (section 3.0.3), and, for remediation handlers, plan support (`supports_plan`, section 4.2). Core registers before any plugin, and the registry refuses a plugin registration that would replace a core step type or another plugin's step type (section 3.0.3). Implementations SHALL register domain-specific handlers in their `register_handlers()` hook (section 6.4). Implementation-registered sieve handlers (non-exhaustive): @@ -2243,7 +2253,7 @@ The `confirm_*` tools are unchanged; approvals are not confirmations. Audit resu ## Appendix C: Removed Requirements -The following requirements have been superseded: by the handler dispatch architecture, by the feature 043 remediation design (the `api_call` and `requires_confirmation` entries), by the operator configuration and trust boundary (the `.baseline.toml` entry, Section 14), and by the removal of unused subsystems (the adapter and `UnifiedLocator` entries, #487). +The following requirements have been superseded: by the handler dispatch architecture, by the feature 043 remediation design (the `api_call` and `requires_confirmation` entries), by the operator configuration and trust boundary (the `.baseline.toml` entry, Section 14), and by the removal of unused subsystems (the adapter, `UnifiedLocator`, and plugin protocol method entries, #487). ### Removed: VerificationPassProtocol **Reason**: Replaced by handler dispatch architecture. Pass classes that implemented this protocol (`DeterministicPass`, `PatternPass`, `LLMPass`, `ManualPass`, `ExecPass`) are superseded by handler functions registered in `SieveHandlerRegistry`. @@ -2332,12 +2342,17 @@ The following requirements have been superseded: by the handler dispatch archite **Reason**: `CheckContext` carried a `UnifiedLocator` that no handler read, and its uncalled `sync_to_project` was a second writer of `.project/` file references beside the remediation file-reference sync (Section 7.9). The control-level `locator` configuration and `use_locator` (Section 11) are unaffected. **Migration**: None. Handlers read `locator.discover` through `use_locator`. +### Removed: ComplianceImplementation control, catalog, and registration methods +**Reason**: No production path called `get_all_controls()`, `get_controls_by_level()`, `get_rules_catalog()`, or `get_remediation_registry()`, and `register_controls()` was required to be a no-op (#487). Controls, SARIF rule metadata, and remediation configuration all come from the framework TOML (section 6.3). The five methods are no longer members of `ComplianceImplementation`, and the framework no longer calls `register_controls()`. +**Migration**: Delete the methods. A plugin that keeps them is still discovered; they are not called. Code that read controls from an implementation loads its framework TOML instead (`load_framework_by_name`, `load_controls_from_framework`). `FrameworkConfig.get_controls_by_level()` and `EffectiveConfig.get_controls_by_level()` are unaffected. + --- ## Version History | Version | Date | Changes | |---------|------|---------| +| 1.0.0-alpha.16 | 2026-10-08 | `ComplianceImplementation` reduced to its five used members; `get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, and `register_controls` removed (Section 6.3; Appendix C; #487). `register_handlers()` is the handler registration hook, `register_sieve_handlers()` accepted only for compatibility, import-time registration not introspectable (Sections 6.4, 12; #451) | | 1.0.0-alpha.15 | 2026-10-08 | Removed the never-dispatched adapter system (`[adapters]`, `[defaults]` adapter keys, control-level `check`, `AdapterRegistry`) and `UnifiedLocator` with its `sync_to_project` writer (Sections 2.1, 6.5, 7.9; Appendix C; #487) | | 1.0.0-alpha.14 | 2026-10-07 | `repro_witness_attestation`: Witness/in-toto runtime-trace verification moves out of `repro_hermetic_build` into its own step type with ceiling `{fail}`, bound to the audited commit and reading only the runtime-trace predicate; a verified clean trace is evidence, not PASS (Sections 3.0.1, 12; #553) | | 1.0.0-alpha.13 | 2026-10-06 | Error class `unexpected_exit`: an `exec` step whose undeclared exit code has no identified cause no longer reports `network`; exit code 127 reports `missing_tool` (Sections 3.3, 5.2; #562) | diff --git a/docs/design/reproducibility-attestation-system.md b/docs/design/reproducibility-attestation-system.md index a0a8e0ed..ed3223d8 100644 --- a/docs/design/reproducibility-attestation-system.md +++ b/docs/design/reproducibility-attestation-system.md @@ -1876,9 +1876,11 @@ reproducibility: ```python # packages/darnit-reproducibility/src/darnit_reproducibility/__init__.py -from darnit.core.plugin import ComplianceImplementation +from importlib.resources import files +from pathlib import Path + -def register() -> ComplianceImplementation: +def register() -> "ReproducibilityImplementation": """Register darnit-reproducibility as a compliance implementation.""" return ReproducibilityImplementation() @@ -1889,79 +1891,77 @@ class ReproducibilityImplementation: def name(self) -> str: return "reproducibility" + @property + def display_name(self) -> str: + return "Scientific Reproducibility Checks" + @property def version(self) -> str: return "0.1.0" - def get_all_controls(self) -> List[ControlSpec]: - """Return reproducibility controls.""" - return [ - ControlSpec( - id="REPRO-ENV-01", - name="Environment Capture", - description="Build environment is fully captured and documented", - level=1, - category="environment", - passes=[ - DeterministicPass( - file_must_exist=[".project/reproducibility.yaml"] - ), - PatternPass( - patterns={"witness_config": r"witness:\s+attestors:"} - ) - ] - ), - ControlSpec( - id="REPRO-BUILD-01", - name="Reproducible Build Definition", - description="Project has reproducible build definition", - level=1, - category="build", - passes=[ - DeterministicPass( - file_must_exist=[ - "Dockerfile", - "Singularity.def", - "flake.nix", - "spack.yaml" - ], - any_of=True - ) - ] - ), - ControlSpec( - id="REPRO-ATTEST-01", - name="Reproducibility Attestation", - description="Build has reproducibility attestation", - level=2, - category="attestation", - passes=[ - DeterministicPass( - file_must_exist=[".attestations/*.intoto.json"] - ), - PatternPass( - patterns={ - "repro_predicate": r"darnit\.dev/attestations/reproducibility" - }, - file_patterns=[".attestations/*.json"] - ) - ] - ), - ControlSpec( - id="REPRO-VERIFY-01", - name="Verified Reproducibility", - description="Build has been independently verified as reproducible", - level=3, - category="verification", - passes=[ - # Check GUAC for independent verification - LLMPass( - prompt="Analyze the reproducibility attestations in GUAC...", - analysis_hints=["Look for independent verifications"] - ) - ] - ) - ] + @property + def spec_version(self) -> str: + return "repro v0.1" + + def get_framework_config_path(self) -> Path | None: + """The controls live in the framework TOML, not in Python.""" + return Path(str(files(__package__) / "reproducibility.toml")) + + def register_handlers(self) -> None: + """Register reproducibility-specific step types (framework-design 6.4).""" + ... +``` + +The controls are declared in `reproducibility.toml`: + +```toml +[controls."REPRO-ENV-01"] +name = "Environment Capture" +description = "Build environment is fully captured and documented" +tags = { level = 1, category = "environment" } + +[[controls."REPRO-ENV-01".passes]] +handler = "file_exists" +files = [".project/reproducibility.yaml"] + +[[controls."REPRO-ENV-01".passes]] +handler = "pattern" +files = [".witness.yaml"] # wherever the project keeps its Witness configuration +patterns = { witness_config = 'witness:\s+attestors:' } + +[controls."REPRO-BUILD-01"] +name = "Reproducible Build Definition" +description = "Project has reproducible build definition" +tags = { level = 1, category = "build" } + +[[controls."REPRO-BUILD-01".passes]] +handler = "file_exists" +files = ["Dockerfile", "Singularity.def", "flake.nix", "spack.yaml"] + +[controls."REPRO-ATTEST-01"] +name = "Reproducibility Attestation" +description = "Build has reproducibility attestation" +tags = { level = 2, category = "attestation" } + +[[controls."REPRO-ATTEST-01".passes]] +handler = "file_exists" +files = [".attestations/*.intoto.json"] + +[[controls."REPRO-ATTEST-01".passes]] +handler = "pattern" +files = [".attestations/*.json"] +patterns = { repro_predicate = 'darnit\.dev/attestations/reproducibility' } + +[controls."REPRO-VERIFY-01"] +name = "Verified Reproducibility" +description = "Build has been independently verified as reproducible" +tags = { level = 3, category = "verification" } + +# Check GUAC for independent verification +[[controls."REPRO-VERIFY-01".passes]] +handler = "llm_eval" +prompt = "Analyze the reproducibility attestations in GUAC..." +analysis_hints = ["Look for independent verifications"] ``` ### Sieve Integration @@ -1971,23 +1971,17 @@ The reproducibility system reuses darnit's sieve pipeline for verification: ```python # Example: Using sieve to verify reproducibility controls -async def verify_reproducibility(local_path: str) -> SieveResult: +def verify_reproducibility(local_path: str) -> list[dict]: """Verify reproducibility controls using the sieve pipeline.""" - - orchestrator = SieveOrchestrator() - impl = ReproducibilityImplementation() - - results = [] - for control in impl.get_all_controls(): - result = await orchestrator.verify( - control, - CheckContext( - local_path=local_path, - project_config=load_project_config(local_path) - ) - ) - results.append(result) - + from darnit.tools.audit import run_sieve_audit + + results, _ = run_sieve_audit( + owner="", + repo="", + local_path=local_path, + default_branch="main", + framework_name="reproducibility", + ) return results ``` diff --git a/docs/getting-started/framework-development.md b/docs/getting-started/framework-development.md index 1c5c2bc3..48a17fb7 100644 --- a/docs/getting-started/framework-development.md +++ b/docs/getting-started/framework-development.md @@ -60,7 +60,7 @@ from darnit_baseline.controls import level1 from darnit.core.discovery import get_implementation impl = get_implementation("openssf-baseline") if impl: - controls = impl.get_all_controls() + config_path = impl.get_framework_config_path() ``` **Why?** This ensures any compliance standard can be implemented as a plugin without modifying the framework. The framework ships with zero knowledge of any specific standard. @@ -84,10 +84,11 @@ sequenceDiagram EP-->>F: List of entry point references F->>I: Call register() function I-->>F: Return ComplianceImplementation instance + F->>I: impl.register_handlers() (if defined) + Note over F,I: Plugin step types and MCP tool handlers registered F->>I: impl.get_framework_config_path() I-->>F: Path to TOML config F->>F: Load and parse TOML controls - F->>I: impl.register_controls() Note over F,I: Framework ready — controls loaded from TOML ``` @@ -99,12 +100,10 @@ The protocol interface (defined in `packages/darnit/src/darnit/core/plugin.py`): | `display_name` | `str` | Human-readable name | | `version` | `str` | Implementation version | | `spec_version` | `str` | Spec version implemented | -| `get_all_controls()` | `list[ControlSpec]` | All controls | -| `get_controls_by_level(n)` | `list[ControlSpec]` | Controls at level n | -| `get_rules_catalog()` | `dict` | SARIF rule definitions | -| `get_remediation_registry()` | `dict` | Auto-fix mappings | | `get_framework_config_path()` | `Path \| None` | TOML config location | -| `register_controls()` | `None` | Register TOML controls | +| `register_handlers()` (optional) | `None` | Register the plugin's sieve step types and MCP tool handlers | + +`register_handlers()` is the one handler hook (framework-design.md 6.4). The framework also calls `register_sieve_handlers()` for compatibility with older out-of-tree plugins; it is not a supported choice for new ones. Registering handlers at module import works, but the framework cannot introspect it. ## The Sieve Pipeline diff --git a/docs/getting-started/implementation-development.md b/docs/getting-started/implementation-development.md index 07151f30..28c6a45c 100644 --- a/docs/getting-started/implementation-development.md +++ b/docs/getting-started/implementation-development.md @@ -207,12 +207,12 @@ def my_handler(config: dict[str, Any], context: HandlerContext) -> HandlerResult ### Registering handlers -Register from your implementation class: +Register from your implementation class's `register_handlers()` method, the hook the framework calls before it loads or audits your framework (framework-design.md 6.4): ```python from darnit.sieve.handler_registry import get_sieve_handler_registry -def register_sieve_handlers(self): +def register_handlers(self): registry = get_sieve_handler_registry() registry.set_plugin_context(self.name) diff --git a/docs/packaging-plugins.md b/docs/packaging-plugins.md index 012baf62..cf93bbb4 100644 --- a/docs/packaging-plugins.md +++ b/docs/packaging-plugins.md @@ -101,12 +101,11 @@ def get_framework_path() -> Path: ## Step 3 — `implementation.py` -The class needs to satisfy the [`ComplianceImplementation` protocol](../packages/darnit/src/darnit/core/plugin.py). For a TOML-only plugin (no Python-defined controls), most methods are stubs. +The class needs to satisfy the [`ComplianceImplementation` protocol](../packages/darnit/src/darnit/core/plugin.py): four identity properties and `get_framework_config_path()`. Controls, SARIF rules, and remediations come from the TOML file, not from the class. `register_handlers()` is optional; it is the hook where a plugin registers its own Python step types and MCP tool handlers (see [The three-layer architecture](#the-three-layer-architecture)). ```python from __future__ import annotations from pathlib import Path -from typing import Any class YourImplementation: @@ -129,26 +128,13 @@ class YourImplementation: def get_framework_config_path(self) -> Path | None: return Path(__file__).parent / "your_framework.toml" - def register_controls(self) -> None: - # No-op for TOML-only plugins. If you write Python control handlers, - # import them here to trigger registration. + def register_handlers(self) -> None: + # Optional. A TOML-only plugin has nothing to register. return None - - def get_all_controls(self) -> list[Any]: - # TOML-defined controls are loaded by the framework via the TOML path. - return [] - - def get_controls_by_level(self, level: int) -> list[Any]: - return [] - - def get_rules_catalog(self) -> dict[str, Any]: - # The framework derives SARIF rules from TOML automatically. - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - return {} ``` +`get_all_controls`, `get_controls_by_level`, `get_rules_catalog`, `get_remediation_registry`, and `register_controls` are no longer part of the protocol (removed in 0.2.0). A plugin that still defines them is discovered as before; the framework does not call them. + The protocol is `@runtime_checkable`, so you can self-test: ```python @@ -342,6 +328,7 @@ See `CLAUDE.md` "Three-Layer Architecture" for the canonical reference. - **TOML control IDs must be unique within your plugin** (the framework deduplicates by ID, with TOML overriding Python registrations). - **CEL backslashes in TOML literal strings are literal.** Use `'\.'` to match a literal dot in a CEL regex, NOT `'\\.'` (the latter produces `\\` + `.` in CEL, which matches "backslash + any char"). - **The framework imports your `register()` lazily** — don't do heavy work at module import time. If you need expensive setup, do it inside `register()` or lazily inside the methods that need it. +- **Register handlers in `register_handlers()`, not at import or in `register()`.** The framework calls `register_handlers()` on every audit and whenever loading your framework needs a step type that is not registered, so your step types survive a registry reset and a cached discovery. Registering at module import works but the framework cannot see it. A method named `register_sieve_handlers()` is still called for compatibility with older plugins; new plugins should not use it. - **The plugin slug (entry-point key) and the `name` property in your implementation must match** — both should equal your framework's slug. If they diverge, some framework tools will see one and some will see the other. --- diff --git a/docs/tutorials/create-new-implementation.md b/docs/tutorials/create-new-implementation.md index a7891498..841f85f6 100644 --- a/docs/tutorials/create-new-implementation.md +++ b/docs/tutorials/create-new-implementation.md @@ -144,9 +144,6 @@ Create `packages/darnit-hygiene/src/darnit_hygiene/implementation.py`: """Code Hygiene compliance implementation for darnit.""" from pathlib import Path -from typing import Any - -from darnit.core.plugin import ControlSpec class HygieneImplementation: @@ -168,34 +165,17 @@ class HygieneImplementation: def spec_version(self) -> str: return "Hygiene v1.0" - def get_all_controls(self) -> list[ControlSpec]: - controls = [] - for level in [1, 2, 3]: - controls.extend(self.get_controls_by_level(level)) - return controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - from darnit.sieve.registry import get_control_registry - - registry = get_control_registry() - return registry.get_specs_by_level(level) - - def get_rules_catalog(self) -> dict[str, Any]: - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - return {} - def get_framework_config_path(self) -> Path | None: # Navigate: implementation.py → darnit_hygiene/ → src/ → darnit-hygiene/ → hygiene.toml return Path(__file__).parent.parent.parent / "hygiene.toml" - def register_controls(self) -> None: - """TOML-first: controls are defined in hygiene.toml. + def register_handlers(self) -> None: + """Optional hook for custom step types and MCP tool handlers. - No Python registration needed for simple implementations. + Every control in hygiene.toml uses built-in step types, so there is + nothing to register. Controls, SARIF rules, and remediations all come + from hygiene.toml. """ - pass ``` ## Step 5: Create the Register Function diff --git a/packages/darnit-baseline/src/darnit_baseline/implementation.py b/packages/darnit-baseline/src/darnit_baseline/implementation.py index 38d0e558..ec81bf38 100644 --- a/packages/darnit-baseline/src/darnit_baseline/implementation.py +++ b/packages/darnit-baseline/src/darnit_baseline/implementation.py @@ -5,9 +5,6 @@ """ from pathlib import Path -from typing import Any - -from darnit.core.plugin import ControlSpec class OSPSBaselineImplementation: @@ -35,98 +32,6 @@ def version(self) -> str: def spec_version(self) -> str: return "OSPS v2026.02.19" - def get_all_controls(self) -> list[ControlSpec]: - """Get all OSPS controls.""" - controls = [] - for level in [1, 2, 3]: - controls.extend(self.get_controls_by_level(level)) - return controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - """Get controls for a specific maturity level.""" - from darnit.config.merger import load_framework_by_name - - config = load_framework_by_name("openssf-baseline") - controls = [] - for control_id, control in config.controls.items(): - ctrl_level = control.level - if ctrl_level is None and control.tags: - ctrl_level = control.tags.get("level") - if ctrl_level == level: - domain = control.domain - if domain is None and control.tags: - domain = control.tags.get("domain", "") - controls.append( - ControlSpec( - control_id=control_id, - name=control.name, - description=control.description or "", - level=level, - domain=domain or (control_id.split("-")[1] if "-" in control_id else "UNKNOWN"), - # Preserve TOML tags on the ControlSpec so downstream - # consumers (e.g., tag-based filtering, feature 025's - # STAGE1-REF-* opt-out from OSPS-format tests) can see - # them. ControlSpec.__post_init__ still adds level/domain. - tags=dict(control.tags) if control.tags else {}, - metadata={ - "full": control.description or "", - "help_uri": control.docs_url - or f"https://baseline.openssf.org/versions/2025-10-10#{control_id}", - }, - ) - ) - return controls - - def get_rules_catalog(self) -> dict[str, Any]: - """Get the rules catalog for SARIF output.""" - from darnit.config.merger import load_framework_by_name - - config = load_framework_by_name("openssf-baseline") - catalog: dict[str, Any] = {} - for control_id, control in config.controls.items(): - level = control.level - if level is None and control.tags: - level = control.tags.get("level", 1) - catalog[control_id] = { - "name": control.name, - "shortDescription": {"text": control.description[:100] if control.description else control.name}, - "level": level or 1, - } - return catalog - - def get_remediation_registry(self) -> dict[str, Any]: - """Get remediation metadata derived from TOML. - - Returns a dict mapping control IDs to their remediation metadata - (safe, requires_api, handler types). - """ - registry: dict[str, Any] = {} - try: - import tomllib - - toml_path = self.get_framework_config_path() - if not toml_path or not toml_path.exists(): - return registry - - with open(toml_path, "rb") as f: - data = tomllib.load(f) - - from darnit.config.framework_schema import FrameworkConfig - - fw = FrameworkConfig(**data) - for cid, control in fw.controls.items(): - if control.remediation and control.remediation.handlers: - handler_types = [h.handler for h in control.remediation.handlers] - registry[cid] = { - "description": control.description or cid, - "safe": control.remediation.safe, - "requires_api": control.remediation.requires_api, - "handler_types": handler_types, - } - except Exception: - pass # Best-effort - return registry - def get_framework_config_path(self) -> Path | None: """Get path to the OpenSSF Baseline framework TOML file. @@ -160,16 +65,6 @@ def get_audit_profiles(self) -> dict | None: return dict(config.audit_profiles) return None - def register_controls(self) -> None: - """No-op. Control definitions come exclusively from TOML. - - This method exists for ComplianceImplementation protocol compatibility. - All control definitions are loaded from openssf-baseline.toml by the - framework's TOML control loader. Plugins should use register_handlers() - to add custom sieve/remediation handlers. - """ - pass - def register_handlers(self) -> None: """Register handlers with the handler registry. diff --git a/packages/darnit-csl/src/darnit_csl/__init__.py b/packages/darnit-csl/src/darnit_csl/__init__.py index 06f4cd48..b29a422c 100644 --- a/packages/darnit-csl/src/darnit_csl/__init__.py +++ b/packages/darnit-csl/src/darnit_csl/__init__.py @@ -63,6 +63,3 @@ def get_optional_framework_path() -> Path: "get_framework_path", "get_optional_framework_path", ] - -# Register custom sieve handlers on import. -from . import handlers # noqa: E402,F401 diff --git a/packages/darnit-csl/src/darnit_csl/handlers.py b/packages/darnit-csl/src/darnit_csl/handlers.py index 5aed8fc7..c11ffe65 100644 --- a/packages/darnit-csl/src/darnit_csl/handlers.py +++ b/packages/darnit-csl/src/darnit_csl/handlers.py @@ -4,9 +4,9 @@ file fails deterministically (no reason to consult the model about a file that is not there), while a file that exists is handed to the LLM for a content-quality judgment. It is registered in the sieve handler registry with the ``llm`` phase so -the orchestrator's PENDING (llm_judgment) branch fires for the present-file case. darnit -imports this package during framework/implementation discovery, so the handler is -available in both the CLI audit path and `darnit serve`. +the orchestrator's PENDING (llm_judgment) branch fires for the present-file case. The +framework registers it through ``CommunitySpecImplementation.register_handlers()`` +(framework-design 6.4). """ from __future__ import annotations @@ -45,7 +45,7 @@ def csl_llm_if_present(config: dict[str, Any], context: HandlerContext) -> Handl ) -def register_sieve_handlers() -> None: +def register_handlers() -> None: """Register the step type under the plugin's name, so the registry records it as this plugin's (feature 044).""" registry = get_sieve_handler_registry() registry.set_plugin_context("community-spec") @@ -60,6 +60,3 @@ def register_sieve_handlers() -> None: ) finally: registry.set_plugin_context(None) - - -register_sieve_handlers() diff --git a/packages/darnit-csl/src/darnit_csl/implementation.py b/packages/darnit-csl/src/darnit_csl/implementation.py index bd72e6cc..ce1fad16 100644 --- a/packages/darnit-csl/src/darnit_csl/implementation.py +++ b/packages/darnit-csl/src/darnit_csl/implementation.py @@ -9,7 +9,6 @@ from __future__ import annotations from pathlib import Path -from typing import Any class CommunitySpecImplementation: @@ -51,24 +50,8 @@ def get_framework_config_path(self) -> Path | None: ) return path - def register_controls(self) -> None: - """No Python-registered controls — everything is in the TOML.""" - return None + def register_handlers(self) -> None: + """Register the ``csl_llm_if_present`` step type.""" + from .handlers import register_handlers - def register_sieve_handlers(self) -> None: - """Register the ``csl_llm_if_present`` step type (again, after a registry reset).""" - from .handlers import register_sieve_handlers - - register_sieve_handlers() - - def get_all_controls(self) -> list[Any]: - return [] - - def get_controls_by_level(self, level: int) -> list[Any]: - return [] - - def get_rules_catalog(self) -> dict[str, Any]: - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - return {} + register_handlers() diff --git a/packages/darnit-example/README.md b/packages/darnit-example/README.md index 68137658..12b40a95 100644 --- a/packages/darnit-example/README.md +++ b/packages/darnit-example/README.md @@ -34,7 +34,7 @@ described in that guide has a concrete counterpart here. | Step 1: Package skeleton | `pyproject.toml`, `src/darnit_example/__init__.py` | | Step 2: Implementation class | `src/darnit_example/implementation.py` | | Step 3: TOML config | `example-hygiene.toml` | -| Step 4: Python controls | `src/darnit_example/controls/level1.py` | +| Step 4: Python handlers | `src/darnit_example/handlers.py` | | Step 5: Remediation | `src/darnit_example/remediation/` | | Step 6: Handler registration | `src/darnit_example/tools.py` | | Step 7: Testing | `tests/darnit_example/` | diff --git a/packages/darnit-example/src/darnit_example/controls/__init__.py b/packages/darnit-example/src/darnit_example/controls/__init__.py deleted file mode 100644 index 741a0e7c..00000000 --- a/packages/darnit-example/src/darnit_example/controls/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Python-defined controls for the Project Hygiene Standard.""" diff --git a/packages/darnit-example/src/darnit_example/controls/level1.py b/packages/darnit-example/src/darnit_example/controls/level1.py deleted file mode 100644 index bc705d1e..00000000 --- a/packages/darnit-example/src/darnit_example/controls/level1.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Python-defined controls for the Project Hygiene Standard. - -All controls are now defined in TOML (example-hygiene.toml) with custom -sieve handlers registered in handlers.py. This file is kept empty for -backward compatibility with any code that imports it. -""" diff --git a/packages/darnit-example/src/darnit_example/implementation.py b/packages/darnit-example/src/darnit_example/implementation.py index c3f0fd82..6d1db3a9 100644 --- a/packages/darnit-example/src/darnit_example/implementation.py +++ b/packages/darnit-example/src/darnit_example/implementation.py @@ -6,53 +6,6 @@ """ from pathlib import Path -from typing import Any - -from darnit.core.plugin import ControlSpec - -# Simple rules catalog for SARIF output -_RULES: dict[str, dict[str, Any]] = { - "PH-DOC-01": { - "name": "ReadmeExists", - "level": 1, - "shortDescription": {"text": "Project has a README file"}, - }, - "PH-DOC-02": { - "name": "LicenseExists", - "level": 1, - "shortDescription": {"text": "Project has a LICENSE file"}, - }, - "PH-DOC-03": { - "name": "ReadmeHasDescription", - "level": 1, - "shortDescription": {"text": "README contains a project description"}, - }, - "PH-SEC-01": { - "name": "SecurityPolicyExists", - "level": 1, - "shortDescription": {"text": "Project has a security policy"}, - }, - "PH-CFG-01": { - "name": "GitignoreExists", - "level": 1, - "shortDescription": {"text": "Project has a .gitignore file"}, - }, - "PH-CFG-02": { - "name": "EditorConfigExists", - "level": 1, - "shortDescription": {"text": "Project has an .editorconfig file"}, - }, - "PH-QA-01": { - "name": "ContributingGuideExists", - "level": 2, - "shortDescription": {"text": "Project has a CONTRIBUTING guide"}, - }, - "PH-CI-01": { - "name": "CIConfigExists", - "level": 2, - "shortDescription": {"text": "Project has CI/CD configuration"}, - }, -} class ExampleHygieneImplementation: @@ -79,40 +32,6 @@ def version(self) -> str: def spec_version(self) -> str: return "PH v1.0" - def get_all_controls(self) -> list[ControlSpec]: - """Get all Project Hygiene controls.""" - controls = [] - for level in [1, 2]: - controls.extend(self.get_controls_by_level(level)) - return controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - """Get controls for a specific maturity level.""" - controls = [] - for rule_id, rule in _RULES.items(): - if rule.get("level") == level: - controls.append( - ControlSpec( - control_id=rule_id, - name=rule.get("name", rule_id), - description=rule.get("shortDescription", {}).get("text", ""), - level=level, - domain=rule_id.split("-")[1] if "-" in rule_id else "UNKNOWN", - metadata={}, - ) - ) - return controls - - def get_rules_catalog(self) -> dict[str, Any]: - """Get the rules catalog for SARIF output.""" - return _RULES - - def get_remediation_registry(self) -> dict[str, Any]: - """Get the remediation registry for auto-fixes.""" - from .remediation.registry import REMEDIATION_REGISTRY - - return REMEDIATION_REGISTRY - def get_framework_config_path(self) -> Path | None: """Get path to the example hygiene framework TOML file. @@ -122,36 +41,31 @@ def get_framework_config_path(self) -> Path | None: # Navigate from implementation.py -> darnit_example -> src -> darnit-example -> toml return Path(__file__).parent.parent.parent / "example-hygiene.toml" - def register_controls(self) -> None: - """Register Python-defined controls with the sieve registry. - - No-op: all controls are now defined in TOML with handler references. - """ - - def register_sieve_handlers(self) -> None: - """Register custom sieve handlers for verification passes.""" + def register_handlers(self) -> None: + """Register the plugin's sieve step types and its MCP tool handlers.""" + from darnit.core.handlers import get_handler_registry from darnit.sieve.handler_registry import get_sieve_handler_registry - from . import handlers + from . import handlers, tools - registry = get_sieve_handler_registry() - registry.set_plugin_context(self.name) + sieve_registry = get_sieve_handler_registry() + sieve_registry.set_plugin_context(self.name) - registry.register( + sieve_registry.register( "readme_description", phase="deterministic", handler_fn=handlers.readme_description_handler, description="Check README has substantive content", settings={"readme_names"}, ) - registry.register( + sieve_registry.register( "readme_quality", phase="pattern", handler_fn=handlers.readme_quality_handler, description="Heuristic check for common README sections", settings={"sections", "min_sections"}, ) - registry.register( + sieve_registry.register( "ci_config", phase="deterministic", handler_fn=handlers.ci_config_handler, @@ -159,13 +73,7 @@ def register_sieve_handlers(self) -> None: settings={"patterns"}, ) - registry.set_plugin_context(None) - - def register_handlers(self) -> None: - """Register handlers with the handler registry.""" - from darnit.core.handlers import get_handler_registry - - from . import tools + sieve_registry.set_plugin_context(None) registry = get_handler_registry() registry.set_plugin_context(self.name) diff --git a/packages/darnit-example/src/darnit_example/remediation/registry.py b/packages/darnit-example/src/darnit_example/remediation/registry.py deleted file mode 100644 index 634f12bc..00000000 --- a/packages/darnit-example/src/darnit_example/remediation/registry.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Remediation registry for the Project Hygiene Standard. - -Maps controls to their automated fix functions. -""" - -from typing import Any - -REMEDIATION_REGISTRY: dict[str, dict[str, Any]] = { - "create_readme": { - "description": "Create a README.md file", - "controls": ["PH-DOC-01", "PH-DOC-03"], - "function": "create_readme", - "safe": True, - "requires_api": False, - }, - "create_gitignore": { - "description": "Create a .gitignore file", - "controls": ["PH-CFG-01"], - "function": "create_gitignore", - "safe": True, - "requires_api": False, - }, -} diff --git a/packages/darnit-example/src/darnit_example/tools.py b/packages/darnit-example/src/darnit_example/tools.py index e66069c1..4168d271 100644 --- a/packages/darnit-example/src/darnit_example/tools.py +++ b/packages/darnit-example/src/darnit_example/tools.py @@ -20,16 +20,11 @@ def _load_all_controls(repo_path: Path, level: int): load_controls_from_effective, load_effective_config_by_name, ) - from darnit.core.discovery import get_implementation from darnit.sieve.registry import get_control_registry config = load_effective_config_by_name("example-hygiene") toml_controls = load_controls_from_effective(config) - impl = get_implementation("example-hygiene") - if impl: - impl.register_controls() - registry = get_control_registry() toml_ids = {c.control_id for c in toml_controls} python_controls = [ diff --git a/packages/darnit-gittuf/src/darnit_gittuf/__init__.py b/packages/darnit-gittuf/src/darnit_gittuf/__init__.py index 9e0e7cff..17db4015 100644 --- a/packages/darnit-gittuf/src/darnit_gittuf/__init__.py +++ b/packages/darnit-gittuf/src/darnit_gittuf/__init__.py @@ -7,10 +7,7 @@ def register() -> GittufImplementation: """Entry point called by darnit plugin discovery.""" - impl = GittufImplementation() - impl.register_controls() - impl.register_sieve_handlers() - return impl + return GittufImplementation() def get_framework_path() -> Path: diff --git a/packages/darnit-gittuf/src/darnit_gittuf/implementation.py b/packages/darnit-gittuf/src/darnit_gittuf/implementation.py index 43947b13..8707a636 100644 --- a/packages/darnit-gittuf/src/darnit_gittuf/implementation.py +++ b/packages/darnit-gittuf/src/darnit_gittuf/implementation.py @@ -1,9 +1,7 @@ """Gittuf plugin implementation for darnit.""" from pathlib import Path -from typing import Any -from darnit.core.plugin import ControlSpec from darnit_gittuf import handlers @@ -30,48 +28,6 @@ def version(self) -> str: def spec_version(self) -> str: return "gittuf v0.1" - def get_all_controls(self) -> list[ControlSpec]: - """Get all controls by collecting across all known levels.""" - all_controls = [] - for level in [1, 2, 3]: - all_controls.extend(self.get_controls_by_level(level)) - return all_controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - all_controls = [ - ControlSpec( - control_id="GT-01.01", - name="GittufInitialized", - description="Repository has Gittuf initialized", - level=1, - domain="GT", - metadata={}, - ), - ControlSpec( - control_id="GT-01.02", - name="GittufPolicyValid", - description="Gittuf policy passes verification", - level=1, - domain="GT", - metadata={}, - ), - ControlSpec( - control_id="GT-02.01", - name="CommitsSigned", - description="Recent commits are cryptographically signed", - level=2, - domain="GT", - metadata={}, - ), - ] - return [c for c in all_controls if c.level == level] - - def get_rules_catalog(self) -> dict[str, Any]: - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - return {} - def get_framework_config_path(self) -> Path | None: from importlib.resources import files @@ -85,10 +41,7 @@ def get_framework_config_path(self) -> Path | None: ) return path - def register_controls(self) -> None: - pass - - def register_sieve_handlers(self) -> None: + def register_handlers(self) -> None: """Register the Gittuf-specific check handlers.""" from darnit.sieve.handler_registry import get_sieve_handler_registry diff --git a/packages/darnit-hello/src/darnit_hello/implementation.py b/packages/darnit-hello/src/darnit_hello/implementation.py index 3276c3c1..56052108 100644 --- a/packages/darnit-hello/src/darnit_hello/implementation.py +++ b/packages/darnit-hello/src/darnit_hello/implementation.py @@ -14,16 +14,14 @@ from __future__ import annotations from pathlib import Path -from typing import Any class HelloImplementation: """A single-control compliance implementation for documentation purposes. The framework discovers this class via the `register()` entry point in - `__init__.py`. At audit time, the framework calls `register_controls()` - (which is a no-op here because controls live in TOML), then queries the - `get_*` methods to enumerate controls and run them. + `__init__.py`. At audit time, it loads the controls from the TOML file + that `get_framework_config_path()` returns and runs them. """ # ---- Required identity properties --------------------------------------- @@ -61,34 +59,17 @@ def get_framework_config_path(self) -> Path | None: """ return Path(__file__).parent / "hello.toml" - def register_controls(self) -> None: - """Register Python-defined controls (none here). + # ---- Optional handler hook ---------------------------------------------- - Implementations with complex pass logic that doesn't fit the built-in - sieve handlers can register Python control functions here via - decorators. This minimal example has no Python controls — the single - control is fully defined in `hello.toml`. + def register_handlers(self) -> None: + """Register this plugin's Python handlers (none here). + + The framework calls this before it loads or audits the `hello` + framework. A plugin with custom checks registers its step types here + with `darnit.sieve.handler_registry.get_sieve_handler_registry()`, and + its MCP tool handlers with `darnit.core.handlers.get_handler_registry()`; + see docs/packaging-plugins.md. Registering at module import instead + works, but the framework cannot see it. This example's single control + uses only built-in step types, so there is nothing to register. """ - # No-op: this example has no Python-registered controls. return None - - def get_all_controls(self) -> list[Any]: - """Return all controls. TOML-defined controls are surfaced by the - framework via the framework config path, not by this method, so the - list here can be empty for TOML-only plugins.""" - return [] - - def get_controls_by_level(self, level: int) -> list[Any]: - """Same convention as get_all_controls — TOML controls are loaded by - the framework, not enumerated by the plugin's Python code.""" - return [] - - def get_rules_catalog(self) -> dict[str, Any]: - """Return the SARIF rules catalog. Empty here; the framework derives - SARIF rules from the TOML config automatically.""" - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - """Return the remediation handler registry. Empty here; this example - has no remediations.""" - return {} diff --git a/packages/darnit-reproducibility/src/darnit_reproducibility/__init__.py b/packages/darnit-reproducibility/src/darnit_reproducibility/__init__.py index 430ac7b8..6eb311b1 100644 --- a/packages/darnit-reproducibility/src/darnit_reproducibility/__init__.py +++ b/packages/darnit-reproducibility/src/darnit_reproducibility/__init__.py @@ -7,10 +7,7 @@ def register() -> ReproducibilityImplementation: """Entry point called by darnit plugin discovery.""" - impl = ReproducibilityImplementation() - impl.register_controls() - impl.register_sieve_handlers() - return impl + return ReproducibilityImplementation() def get_framework_path() -> Path: diff --git a/packages/darnit-reproducibility/src/darnit_reproducibility/implementation.py b/packages/darnit-reproducibility/src/darnit_reproducibility/implementation.py index 2a50a2af..8b3d97b0 100644 --- a/packages/darnit-reproducibility/src/darnit_reproducibility/implementation.py +++ b/packages/darnit-reproducibility/src/darnit_reproducibility/implementation.py @@ -1,9 +1,7 @@ """Scientific reproducibility plugin implementation for darnit.""" from pathlib import Path -from typing import Any -from darnit.core.plugin import ControlSpec from darnit_reproducibility import handlers @@ -31,65 +29,6 @@ def version(self) -> str: def spec_version(self) -> str: return "repro v0.1" - def get_all_controls(self) -> list[ControlSpec]: - controls = [] - for level in [1, 2, 3]: - controls.extend(self.get_controls_by_level(level)) - return controls - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - all_controls = [ - ControlSpec( - control_id="RE-01.01", - name="DependenciesPinned", - description="Dependencies are pinned to exact versions with checksums", - level=1, - domain="RE", - metadata={}, - ), - ControlSpec( - control_id="RE-01.02", - name="BuildEnvDeclared", - description="Build environment is explicitly declared", - level=1, - domain="RE", - metadata={}, - ), - ControlSpec( - control_id="RE-02.01", - name="HermeticBuild", - description="Build does not fetch dependencies at build time", - level=2, - domain="RE", - metadata={}, - ), - ControlSpec( - control_id="RE-02.02", - name="ProvenanceExists", - description="Build produces a signed provenance attestation", - level=2, - domain="RE", - metadata={}, - ), - ControlSpec( - control_id="RE-03.01", - name="BitForBitReproducible", - description="Build output is identical across independent builds", - level=3, - domain="RE", - metadata={}, - ), - ] - return [c for c in all_controls if c.level == level] - - def get_rules_catalog(self) -> dict[str, Any]: - return {} - - def get_remediation_registry(self) -> dict[str, Any]: - # Intentionally empty: TOML is the source of truth for remediation. - # RemediationExecutor reads FrameworkConfig.controls[id].remediation directly. - return {} - def get_framework_config_path(self) -> Path | None: from importlib.resources import files @@ -103,10 +42,7 @@ def get_framework_config_path(self) -> Path | None: ) return path - def register_controls(self) -> None: - pass - - def register_sieve_handlers(self) -> None: + def register_handlers(self) -> None: """Register the reproducibility-specific check handlers.""" from darnit.sieve.handler_registry import get_sieve_handler_registry diff --git a/packages/darnit/README.md b/packages/darnit/README.md index bcc1d1c3..9ecb866f 100644 --- a/packages/darnit/README.md +++ b/packages/darnit/README.md @@ -28,7 +28,7 @@ from darnit.core.discovery import get_implementation # Get a compliance implementation by name impl = get_implementation("openssf-baseline") if impl: - controls = impl.get_all_controls() + config_path = impl.get_framework_config_path() ``` ### Configuration Management diff --git a/packages/darnit/src/darnit/__init__.py b/packages/darnit/src/darnit/__init__.py index 71d90ee1..3ee9a7ed 100644 --- a/packages/darnit/src/darnit/__init__.py +++ b/packages/darnit/src/darnit/__init__.py @@ -8,7 +8,7 @@ impl = get_implementation("openssf-baseline") if impl: - controls = impl.get_all_controls() + config_path = impl.get_framework_config_path() """ __version__ = "0.1.0" diff --git a/packages/darnit/src/darnit/core/discovery.py b/packages/darnit/src/darnit/core/discovery.py index d506fb24..de9708d8 100644 --- a/packages/darnit/src/darnit/core/discovery.py +++ b/packages/darnit/src/darnit/core/discovery.py @@ -133,7 +133,8 @@ def register_implementation_handlers(framework_name: str | None) -> bool: Returns: True if handlers were registered, False if there was nothing to do (no framework name, no such implementation, or the implementation - exposes neither ``register_sieve_handlers`` nor ``register_handlers``). + exposes neither ``register_handlers`` nor the compatibility name + ``register_sieve_handlers``). """ if not framework_name: return False @@ -143,18 +144,14 @@ def register_implementation_handlers(framework_name: str | None) -> bool: logger.debug("No implementation found for '%s'", framework_name) return False - # Two method names are in use across in-tree plugins: - # register_handlers -- documented in CLAUDE.md; darnit-baseline - # register_sieve_handlers -- darnit-gittuf, darnit-reproducibility - # Those two work today only because their `register()` entry point calls - # register_sieve_handlers() during discovery. That is a side channel, not - # the protocol: discovery results are cached, so any caller that warmed - # the cache earlier in the process leaves the handlers unregistered and - # every plugin control silently falls through to `manual`. Accepting both - # names here makes registration explicit and cache-independent. A plugin - # may define both (darnit-example registers its step types in one and its - # MCP tools in the other), so every one present is called. - # hasattr per Constitution Principle I: missing methods degrade, never crash. + # register_handlers() is the protocol hook (framework-design 6.4, #451). + # register_sieve_handlers is accepted for compatibility with out-of-tree + # plugins written before the hook was standardized; it is not a supported + # choice, and no in-tree plugin uses it. A plugin defining both has each + # called. Calling here, rather than relying on a plugin's register() + # entry point, keeps registration independent of the discovery cache + # (#427). hasattr per Constitution Principle I: missing methods degrade, + # never crash. methods = [getattr(impl, name) for name in ("register_sieve_handlers", "register_handlers") if hasattr(impl, name)] if not methods: return False diff --git a/packages/darnit/src/darnit/core/plugin.py b/packages/darnit/src/darnit/core/plugin.py index 829672c8..0156ef11 100644 --- a/packages/darnit/src/darnit/core/plugin.py +++ b/packages/darnit/src/darnit/core/plugin.py @@ -54,7 +54,7 @@ class OSPSBaselineImplementation: version = "0.1.0" spec_version = "OSPS v2025.10.10" - def get_all_controls(self) -> List[ControlSpec]: + def get_framework_config_path(self) -> Path | None: ... """ @@ -78,22 +78,6 @@ def spec_version(self) -> str: """Specification version this implements (e.g., 'OSPS v2025.10.10').""" ... - def get_all_controls(self) -> list[ControlSpec]: - """Get all controls defined by this implementation.""" - ... - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - """Get controls for a specific maturity level.""" - ... - - def get_rules_catalog(self) -> dict[str, Any]: - """Get the rules catalog for SARIF output.""" - ... - - def get_remediation_registry(self) -> dict[str, Any]: - """Get the remediation registry for auto-fixes.""" - ... - def get_framework_config_path(self) -> Path | None: """Get path to the framework configuration file (e.g., TOML). @@ -111,21 +95,6 @@ def get_framework_config_path(self) -> Path | None: """ ... - def register_controls(self) -> None: - """Register this implementation's Python-defined controls. - - Implementations should import their control modules here to trigger - registration via decorators (e.g., @register_control). This allows - the framework to load controls without knowing implementation-specific - module paths. - - Example: - # In openssf-baseline implementation: - def register_controls(self) -> None: - from .controls import level1, level2, level3 # noqa: F401 - """ - ... - # ------------------------------------------------------------------------- # Optional action handlers (NOT part of the Protocol — checked via hasattr) # @@ -133,6 +102,10 @@ def register_controls(self) -> None: # actions. The framework calls hasattr() before invoking them, so a plugin # that only does checks does not need to implement the others. # + # def register_handlers(self) -> None: ... + # The handler hook: registers sieve step types and MCP tool handlers + # (framework-design 6.4). register_sieve_handlers() is still called + # for compatibility, but is not a supported name for new plugins. # def get_check_handlers(self) -> dict[str, Any]: ... # def get_context_handlers(self) -> dict[str, Any]: ... # def get_remediation_handlers(self) -> dict[str, Any]: ... diff --git a/packages/darnit/src/darnit/server/tools/builtin_audit.py b/packages/darnit/src/darnit/server/tools/builtin_audit.py index 61f8bc75..833af0a3 100644 --- a/packages/darnit/src/darnit/server/tools/builtin_audit.py +++ b/packages/darnit/src/darnit/server/tools/builtin_audit.py @@ -90,13 +90,7 @@ async def builtin_audit( except Exception as e: return f"Error loading controls: {e}" - # Register and load Python-defined controls if implementation exists impl = get_implementation(_framework_name) - if impl and hasattr(impl, "register_controls"): - try: - impl.register_controls() - except Exception as e: - logger.warning(f"Error registering Python controls: {e}") # Merge Python-defined controls that aren't in TOML registry = get_control_registry() diff --git a/packages/darnit/src/darnit/server/tools/builtin_list.py b/packages/darnit/src/darnit/server/tools/builtin_list.py index 88aaead2..2105e7b7 100644 --- a/packages/darnit/src/darnit/server/tools/builtin_list.py +++ b/packages/darnit/src/darnit/server/tools/builtin_list.py @@ -37,7 +37,6 @@ async def builtin_list_controls( load_controls_from_effective, load_effective_config_by_name, ) - from darnit.core.discovery import get_implementation from darnit.sieve.registry import get_control_registry if not _framework_name: @@ -55,14 +54,6 @@ async def builtin_list_controls( except Exception as e: return f"Error loading controls: {e}" - # Register Python controls - impl = get_implementation(_framework_name) - if impl and hasattr(impl, "register_controls"): - try: - impl.register_controls() - except Exception as e: - logger.warning(f"Error registering Python controls: {e}") - registry = get_control_registry() toml_ids = {c.control_id for c in toml_controls} python_controls = [ diff --git a/packages/darnit/src/darnit/tools/audit.py b/packages/darnit/src/darnit/tools/audit.py index 6295bf6f..ed76c33b 100644 --- a/packages/darnit/src/darnit/tools/audit.py +++ b/packages/darnit/src/darnit/tools/audit.py @@ -815,22 +815,6 @@ def run_sieve_audit( if controls is not None: all_controls = list(controls) else: - # Register Python-defined controls via plugin system - if resolved_fw: - try: - from darnit.core.discovery import get_implementation - - impl = get_implementation(resolved_fw) - if impl and hasattr(impl, "register_controls"): - impl.register_controls() - logger.debug(f"Registered Python control definitions from {impl.name}") - elif impl: - logger.debug(f"Implementation {impl.name} does not provide register_controls()") - else: - logger.debug("Implementation '%s' not found for control registration", resolved_fw) - except Exception as e: - logger.debug(f"Python control modules not available: {e}") - # Register controls from TOML framework definition (primary source of truth) _register_toml_controls(resolved_fw) diff --git a/tests/darnit/config/test_expr_references.py b/tests/darnit/config/test_expr_references.py index 01c5e5b6..d25b5307 100644 --- a/tests/darnit/config/test_expr_references.py +++ b/tests/darnit/config/test_expr_references.py @@ -94,19 +94,21 @@ def test_step_type_without_expression_names(self) -> None: @pytest.fixture(scope="module") def plugin_step_types() -> None: - import importlib - - import darnit_csl.handlers + from darnit_csl.implementation import CommunitySpecImplementation from darnit_gittuf.implementation import GittufImplementation from darnit_reproducibility.implementation import ReproducibilityImplementation from darnit_baseline.implementation import OSPSBaselineImplementation from darnit_example.implementation import ExampleHygieneImplementation - importlib.reload(darnit_csl.handlers) - OSPSBaselineImplementation().register_handlers() - for implementation in (ExampleHygieneImplementation, GittufImplementation, ReproducibilityImplementation): - implementation().register_sieve_handlers() + for implementation in ( + OSPSBaselineImplementation, + CommunitySpecImplementation, + ExampleHygieneImplementation, + GittufImplementation, + ReproducibilityImplementation, + ): + implementation().register_handlers() @pytest.mark.unit diff --git a/tests/darnit/config/test_strict_framework_loading.py b/tests/darnit/config/test_strict_framework_loading.py index 5d560e30..47749e05 100644 --- a/tests/darnit/config/test_strict_framework_loading.py +++ b/tests/darnit/config/test_strict_framework_loading.py @@ -58,19 +58,21 @@ def _shipped_tomls() -> list[Path]: @pytest.fixture(scope="module") def plugin_step_types() -> None: - import importlib - - import darnit_csl.handlers + from darnit_csl.implementation import CommunitySpecImplementation from darnit_gittuf.implementation import GittufImplementation from darnit_reproducibility.implementation import ReproducibilityImplementation from darnit_baseline.implementation import OSPSBaselineImplementation from darnit_example.implementation import ExampleHygieneImplementation - importlib.reload(darnit_csl.handlers) - OSPSBaselineImplementation().register_handlers() - for implementation in (ExampleHygieneImplementation, GittufImplementation, ReproducibilityImplementation): - implementation().register_sieve_handlers() + for implementation in ( + OSPSBaselineImplementation, + CommunitySpecImplementation, + ExampleHygieneImplementation, + GittufImplementation, + ReproducibilityImplementation, + ): + implementation().register_handlers() def _loose(config, context: HandlerContext) -> HandlerResult: # noqa: ARG001 diff --git a/tests/darnit/core/test_plugin.py b/tests/darnit/core/test_plugin.py index defd59b6..a48f7489 100644 --- a/tests/darnit/core/test_plugin.py +++ b/tests/darnit/core/test_plugin.py @@ -15,43 +15,14 @@ class FullyCompliantImplementation: version = "0.1.0" spec_version = "TEST v1" - def get_all_controls(self) -> list[ControlSpec]: - return [ - ControlSpec( - control_id="TEST-01", - name="Test control", - description="A test control", - level=1, - domain="TEST", - metadata={}, - ) - ] - - def get_controls_by_level(self, level: int) -> list[ControlSpec]: - return [ - control for control in self.get_all_controls() if control.level == level - ] - - def get_rules_catalog(self) -> dict[str, str]: - return {"version": self.spec_version} - - def get_remediation_registry(self) -> dict[str, str]: - return {"test": "handler"} - def get_framework_config_path(self) -> Path | None: return Path("/tmp/framework.toml") - def register_controls(self) -> None: - return None +class LegacyMethodsImplementation(FullyCompliantImplementation): + """A plugin written before #487 that still defines the removed methods.""" -class MissingRegisterControlsImplementation: - """Deliberately incomplete implementation for protocol checks.""" - - name = "broken-framework" - display_name = "Broken Framework" - version = "0.1.0" - spec_version = "TEST v1" + name = "legacy-framework" def get_all_controls(self) -> list[ControlSpec]: return [] @@ -65,10 +36,19 @@ def get_rules_catalog(self) -> dict[str, str]: def get_remediation_registry(self) -> dict[str, str]: return {} - def get_framework_config_path(self) -> Path | None: + def register_controls(self) -> None: return None +class MissingConfigPathImplementation: + """Deliberately incomplete implementation for protocol checks.""" + + name = "broken-framework" + display_name = "Broken Framework" + version = "0.1.0" + spec_version = "TEST v1" + + class TestControlSpec: """Tests for ControlSpec dataclass behavior.""" @@ -116,18 +96,67 @@ class TestComplianceImplementationProtocol: @pytest.mark.unit def test_runtime_check_accepts_complete_implementation(self): - """A class implementing the full protocol satisfies isinstance().""" + """A class implementing the five required members satisfies isinstance().""" implementation = FullyCompliantImplementation() assert isinstance(implementation, ComplianceImplementation) assert implementation.get_framework_config_path() == Path( "/tmp/framework.toml" ) - assert implementation.get_all_controls()[0].control_id == "TEST-01" + + @pytest.mark.unit + def test_runtime_check_accepts_implementation_with_removed_methods(self): + """Methods removed from the protocol (#487) do not affect isinstance().""" + assert isinstance(LegacyMethodsImplementation(), ComplianceImplementation) @pytest.mark.unit def test_runtime_check_rejects_missing_required_method(self): """A class missing a required protocol method fails isinstance().""" - implementation = MissingRegisterControlsImplementation() + implementation = MissingConfigPathImplementation() assert not isinstance(implementation, ComplianceImplementation) + + +class _EntryPoint: + def __init__(self, name: str, factory: type) -> None: + self.name = name + self.module = f"{name.replace('-', '_')}_pkg" + self._factory = factory + + def load(self): + return self._factory + + +class TestDiscoveryOfTrimmedProtocol: + """#487: dropping protocol members must not drop plugins out of discovery.""" + + @pytest.fixture(autouse=True) + def _fake_entry_points(self, monkeypatch: pytest.MonkeyPatch): + import importlib.metadata + + from darnit.core.discovery import clear_cache + + eps = [ + _EntryPoint("test-framework", FullyCompliantImplementation), + _EntryPoint("legacy-framework", LegacyMethodsImplementation), + _EntryPoint("broken-framework", MissingConfigPathImplementation), + ] + monkeypatch.setattr(importlib.metadata, "entry_points", lambda **_: eps) + clear_cache() + yield + clear_cache() + + @pytest.mark.unit + def test_plugin_with_and_without_removed_methods_is_discovered(self): + from darnit.core.discovery import discover_implementations + + found = discover_implementations() + + assert isinstance(found.get("test-framework"), FullyCompliantImplementation) + assert isinstance(found.get("legacy-framework"), LegacyMethodsImplementation) + + @pytest.mark.unit + def test_plugin_missing_a_required_member_is_not_discovered(self): + from darnit.core.discovery import discover_implementations + + assert "broken-framework" not in discover_implementations() diff --git a/tests/darnit/sieve/test_handler_authority_regression.py b/tests/darnit/sieve/test_handler_authority_regression.py index 78a8a34f..d41d7f52 100644 --- a/tests/darnit/sieve/test_handler_authority_regression.py +++ b/tests/darnit/sieve/test_handler_authority_regression.py @@ -101,7 +101,7 @@ class TestPluginHandlersAreDispositive: def test_every_expected_dispositive_handler_is_dispositive(self) -> None: # Force plugin registration by calling each implementation's - # register_sieve_handlers / register_handlers method. + # register_handlers method. self._register_all_known_plugin_handlers() registry = get_sieve_handler_registry() @@ -118,7 +118,7 @@ def test_every_expected_dispositive_handler_is_dispositive(self) -> None: "PASS terminates the Check phase. Fix by adding " "`ceiling={\"pass\", \"fail\"}` to the handler's " "`registry.register(...)` call in its plugin's " - "`register_sieve_handlers()`." + "`register_handlers()`." ) @staticmethod @@ -135,7 +135,7 @@ def _register_all_known_plugin_handlers() -> None: GittufImplementation, ) - GittufImplementation().register_sieve_handlers() + GittufImplementation().register_handlers() except Exception: pass @@ -145,7 +145,7 @@ def _register_all_known_plugin_handlers() -> None: ReproducibilityImplementation, ) - ReproducibilityImplementation().register_sieve_handlers() + ReproducibilityImplementation().register_handlers() except Exception: pass diff --git a/tests/darnit/sieve/test_handler_registry_metadata.py b/tests/darnit/sieve/test_handler_registry_metadata.py index daed47a4..cbf04f41 100644 --- a/tests/darnit/sieve/test_handler_registry_metadata.py +++ b/tests/darnit/sieve/test_handler_registry_metadata.py @@ -63,19 +63,21 @@ def _noop(config, context): # noqa: ARG001 @pytest.fixture(scope="module") def plugin_step_types() -> None: - import importlib - - import darnit_csl.handlers + from darnit_csl.implementation import CommunitySpecImplementation from darnit_gittuf.implementation import GittufImplementation from darnit_reproducibility.implementation import ReproducibilityImplementation from darnit_baseline.implementation import OSPSBaselineImplementation from darnit_example.implementation import ExampleHygieneImplementation - importlib.reload(darnit_csl.handlers) # registers on import, possibly into a registry reset since - OSPSBaselineImplementation().register_handlers() - for implementation in (ExampleHygieneImplementation, GittufImplementation, ReproducibilityImplementation): - implementation().register_sieve_handlers() + for implementation in ( + OSPSBaselineImplementation, + CommunitySpecImplementation, + ExampleHygieneImplementation, + GittufImplementation, + ReproducibilityImplementation, + ): + implementation().register_handlers() class TestRegister: diff --git a/tests/darnit/test_plugin_handler_registration.py b/tests/darnit/test_plugin_handler_registration.py index 3d28482b..6b352e78 100644 --- a/tests/darnit/test_plugin_handler_registration.py +++ b/tests/darnit/test_plugin_handler_registration.py @@ -21,9 +21,7 @@ from darnit.core.discovery import register_implementation_handlers from darnit.sieve.handler_registry import get_sieve_handler_registry -# Handlers shipped by darnit-reproducibility. It is the in-tree plugin that -# exercises the `register_sieve_handlers` spelling (see the naming note in -# TestProtocolMethodNaming below). +# Handlers shipped by darnit-reproducibility, the plugin #427 was reported against. _REPRO_HANDLERS = ( "repro_deps_pinned", "repro_build_env_declared", @@ -47,17 +45,16 @@ def test_registers_reproducibility_handlers(self) -> None: @pytest.mark.unit def test_registers_baseline_handlers(self) -> None: - """darnit-baseline uses the other spelling; both must work.""" assert register_implementation_handlers("openssf-baseline") is True @pytest.mark.unit def test_works_when_discovery_cache_is_already_warm(self) -> None: """The regression that made #427 intermittent. - `discover_implementations()` is cached, and the plugins that - self-register do so from their `register()` entry point -- which - runs once, during the first discovery. Any caller that warmed the - cache earlier in the process therefore left handlers unregistered. + `discover_implementations()` is cached, and plugins that registered + from their `register()` entry point did so once, during the first + discovery. Any caller that warmed the cache earlier in the process + therefore left handlers unregistered. That timing dependence is why the bug presented as "3 of 5 handlers missing" on one run and "5 of 5" on the next. @@ -103,19 +100,76 @@ def test_plugin_exception_is_contained(self) -> None: class TestProtocolMethodNaming: - """Both in-tree spellings of the registration method must be honored. + """`register_handlers()` is the hook; `register_sieve_handlers()` is a compatibility shim (#451). - `CLAUDE.md` documents `register_handlers()`, and darnit-baseline - implements it. darnit-gittuf and darnit-reproducibility implement - `register_sieve_handlers()` instead. Until those converge, the - framework accepts either -- otherwise two of four in-tree plugins - register nothing. - - Reconciling the plugins onto one name is tracked separately; this - test pins current behavior so the tolerance is deliberate and - visible rather than accidental. + Every in-tree plugin uses `register_handlers()`. The framework still + calls `register_sieve_handlers()` so out-of-tree plugins written + against the old name keep their step types. """ + @pytest.mark.unit + @pytest.mark.parametrize( + ("module", "cls"), + [ + ("darnit_baseline.implementation", "OSPSBaselineImplementation"), + ("darnit_csl.implementation", "CommunitySpecImplementation"), + ("darnit_example.implementation", "ExampleHygieneImplementation"), + ("darnit_gittuf.implementation", "GittufImplementation"), + ("darnit_hello.implementation", "HelloImplementation"), + ("darnit_reproducibility.implementation", "ReproducibilityImplementation"), + ], + ) + def test_in_tree_plugins_use_only_register_handlers(self, module: str, cls: str) -> None: + import importlib + + implementation = getattr(importlib.import_module(module), cls) + assert callable(getattr(implementation, "register_handlers", None)) + assert not hasattr(implementation, "register_sieve_handlers") + + @pytest.mark.unit + def test_plugin_with_only_the_compatibility_name_registers_its_step_types( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An out-of-tree plugin that predates #451 still gets its step types registered.""" + from unittest.mock import patch + + import darnit.sieve.handler_registry as handler_registry + from darnit.sieve.handler_registry import HandlerResult, HandlerResultStatus + + monkeypatch.setattr(handler_registry, "_sieve_handler_registry", None) + + class LegacyPlugin: + name = "legacy" + + def register_sieve_handlers(self) -> None: + registry = get_sieve_handler_registry() + registry.set_plugin_context(self.name) + registry.register( + "legacy_check", + phase="deterministic", + handler_fn=lambda config, context: HandlerResult(status=HandlerResultStatus.FAIL, message="x"), + ceiling={"fail"}, + settings=frozenset(), + ) + registry.set_plugin_context(None) + + with patch("darnit.core.discovery.get_implementation", return_value=LegacyPlugin()): + assert register_implementation_handlers("legacy") is True + info = get_sieve_handler_registry().get("legacy_check") + assert info is not None + assert info.plugin == "legacy" + + @pytest.mark.unit + def test_csl_step_type_registers_through_the_hook(self, monkeypatch: pytest.MonkeyPatch) -> None: + """darnit-csl registered at import before #451; a registry reset must not lose its step type.""" + import darnit.sieve.handler_registry as handler_registry + + monkeypatch.setattr(handler_registry, "_sieve_handler_registry", None) + assert get_sieve_handler_registry().get("csl_llm_if_present") is None + + assert register_implementation_handlers("community-spec") is True + assert get_sieve_handler_registry().get("csl_llm_if_present") is not None + @pytest.mark.unit def test_accepts_register_handlers_spelling(self) -> None: from unittest.mock import MagicMock, patch @@ -136,7 +190,7 @@ def test_accepts_register_sieve_handlers_spelling(self) -> None: @pytest.mark.unit def test_implementation_with_both_spellings_calls_both(self) -> None: - """darnit-example registers MCP tools in one and step types in the other (044 review).""" + """A plugin may define both names; neither is skipped.""" from unittest.mock import MagicMock, patch impl = MagicMock(spec=["register_handlers", "register_sieve_handlers"]) diff --git a/tests/darnit_baseline/test_implementation.py b/tests/darnit_baseline/test_implementation.py index f14ccb97..b50047cb 100644 --- a/tests/darnit_baseline/test_implementation.py +++ b/tests/darnit_baseline/test_implementation.py @@ -8,6 +8,14 @@ from darnit_baseline.implementation import OSPSBaselineImplementation +def _framework_controls(impl): + """Controls as the audit loads them, from the implementation's framework TOML.""" + from darnit.config import load_controls_from_framework + from darnit.config.merger import load_framework_config + + return load_controls_from_framework(load_framework_config(impl.get_framework_config_path())) + + class TestOSPSBaselineImplementation: """Tests for OSPSBaselineImplementation class.""" @@ -30,35 +38,12 @@ def test_is_compliance_implementation(self, impl): assert isinstance(impl, ComplianceImplementation) @pytest.mark.unit - def test_get_all_controls(self, impl): - """Test get_all_controls returns all controls.""" - controls = impl.get_all_controls() + def test_framework_toml_has_controls_at_every_level(self, impl): + """The framework TOML defines controls at all three levels.""" + controls = _framework_controls(impl) assert len(controls) > 0 - # Should have controls from all 3 levels levels = {c.level for c in controls} - assert 1 in levels - assert 2 in levels - assert 3 in levels - - @pytest.mark.unit - def test_get_controls_by_level(self, impl): - """Test get_controls_by_level filters correctly.""" - level1 = impl.get_controls_by_level(1) - level2 = impl.get_controls_by_level(2) - level3 = impl.get_controls_by_level(3) - - # All should be non-empty - assert len(level1) > 0 - assert len(level2) > 0 - assert len(level3) > 0 - - # All controls should have correct level - assert all(c.level == 1 for c in level1) - assert all(c.level == 2 for c in level2) - assert all(c.level == 3 for c in level3) - - # Sum should equal total - assert len(level1) + len(level2) + len(level3) == len(impl.get_all_controls()) + assert {1, 2, 3} <= levels @pytest.mark.unit def test_control_ids_are_osps_format(self, impl): @@ -69,7 +54,7 @@ def test_control_ids_are_osps_format(self, impl): with the OSPS-* set; the format check applies only to controls that are not explicitly marked as stage-reference fixtures. """ - controls = impl.get_all_controls() + controls = _framework_controls(impl) for control in controls: # Skip stage-reference controls; they use STAGE1-REF-* naming. tags = control.tags or {} @@ -85,18 +70,11 @@ def test_control_ids_are_osps_format(self, impl): @pytest.mark.unit def test_control_domains(self, impl): """Test controls have valid domains.""" - controls = impl.get_all_controls() + controls = _framework_controls(impl) valid_domains = {"AC", "BR", "DO", "GV", "LE", "QA", "SA", "VM"} for control in controls: assert control.domain in valid_domains, f"Invalid domain: {control.domain}" - @pytest.mark.unit - def test_rules_catalog_has_required_fields(self, impl): - """Test rules catalog entries have required fields.""" - catalog = impl.get_rules_catalog() - for rule_id, rule in catalog.items(): - assert "name" in rule or "shortDescription" in rule - assert "level" in rule class TestHandlerRegistration: """Tests for auto-registration of handlers.""" diff --git a/tests/darnit_example/test_implementation.py b/tests/darnit_example/test_implementation.py index 99cd45f5..aed370c4 100644 --- a/tests/darnit_example/test_implementation.py +++ b/tests/darnit_example/test_implementation.py @@ -7,6 +7,14 @@ from darnit_example.implementation import ExampleHygieneImplementation +def _framework_controls(impl): + """Controls as the audit loads them, from the implementation's framework TOML.""" + from darnit.config import load_controls_from_framework + from darnit.config.merger import load_framework_config + + return load_controls_from_framework(load_framework_config(impl.get_framework_config_path())) + + class TestExampleHygieneImplementation: """Tests for ExampleHygieneImplementation class.""" @@ -26,28 +34,15 @@ def test_is_compliance_implementation(self, impl): assert isinstance(impl, ComplianceImplementation) @pytest.mark.unit - def test_get_all_controls(self, impl): - controls = impl.get_all_controls() + def test_framework_toml_controls(self, impl): + controls = _framework_controls(impl) assert len(controls) == 8 - levels = {c.level for c in controls} - assert levels == {1, 2} - - @pytest.mark.unit - def test_get_controls_by_level(self, impl): - level1 = impl.get_controls_by_level(1) - level2 = impl.get_controls_by_level(2) - - assert len(level1) == 6 - assert len(level2) == 2 - - assert all(c.level == 1 for c in level1) - assert all(c.level == 2 for c in level2) - - assert len(level1) + len(level2) == len(impl.get_all_controls()) + assert sum(c.level == 1 for c in controls) == 6 + assert sum(c.level == 2 for c in controls) == 2 @pytest.mark.unit def test_control_ids_are_ph_format(self, impl): - controls = impl.get_all_controls() + controls = _framework_controls(impl) for control in controls: assert control.control_id.startswith("PH-") parts = control.control_id.split("-") @@ -55,23 +50,11 @@ def test_control_ids_are_ph_format(self, impl): @pytest.mark.unit def test_control_domains(self, impl): - controls = impl.get_all_controls() + controls = _framework_controls(impl) valid_domains = {"DOC", "SEC", "CFG", "QA", "CI"} for control in controls: assert control.domain in valid_domains, f"Invalid domain: {control.domain}" - @pytest.mark.unit - def test_get_rules_catalog(self, impl): - catalog = impl.get_rules_catalog() - assert isinstance(catalog, dict) - assert len(catalog) == 8 - - @pytest.mark.unit - def test_get_remediation_registry(self, impl): - registry = impl.get_remediation_registry() - assert isinstance(registry, dict) - assert len(registry) > 0 - @pytest.mark.unit def test_get_framework_config_path(self, impl): path = impl.get_framework_config_path() diff --git a/tests/darnit_gittuf/test_implementation.py b/tests/darnit_gittuf/test_implementation.py index b5c34b97..a80f1cde 100644 --- a/tests/darnit_gittuf/test_implementation.py +++ b/tests/darnit_gittuf/test_implementation.py @@ -4,6 +4,14 @@ from darnit_gittuf.implementation import GittufImplementation +def _framework_controls(impl): + """Controls as the audit loads them, from the implementation's framework TOML.""" + from darnit.config import load_controls_from_framework + from darnit.config.merger import load_framework_config + + return load_controls_from_framework(load_framework_config(impl.get_framework_config_path())) + + class TestGittufImplementation: """Tests that GittufImplementation satisfies the plugin protocol.""" @@ -20,30 +28,19 @@ def test_version_is_string(self) -> None: assert isinstance(self.impl.version, str) assert len(self.impl.version) > 0 - def test_get_all_controls_returns_three(self) -> None: - controls = self.impl.get_all_controls() + def test_framework_toml_defines_three_controls(self) -> None: + controls = _framework_controls(self.impl) assert len(controls) == 3 def test_control_ids(self) -> None: - ids = {c.control_id for c in self.impl.get_all_controls()} + ids = {c.control_id for c in _framework_controls(self.impl)} assert "GT-01.01" in ids assert "GT-01.02" in ids assert "GT-02.01" in ids - def test_level_1_controls(self) -> None: - controls = self.impl.get_controls_by_level(1) - assert len(controls) == 2 - for c in controls: - assert c.level == 1 - - def test_level_2_controls(self) -> None: - controls = self.impl.get_controls_by_level(2) - assert len(controls) == 1 - assert controls[0].control_id == "GT-02.01" - - def test_level_3_returns_empty(self) -> None: - controls = self.impl.get_controls_by_level(3) - assert controls == [] + def test_control_levels(self) -> None: + levels = {c.control_id: c.level for c in _framework_controls(self.impl)} + assert levels == {"GT-01.01": 1, "GT-01.02": 1, "GT-02.01": 2} def test_framework_config_path_exists(self) -> None: path = self.impl.get_framework_config_path() diff --git a/tests/darnit_reproducibility/test_implementation.py b/tests/darnit_reproducibility/test_implementation.py index 65bda663..1c57bcd8 100644 --- a/tests/darnit_reproducibility/test_implementation.py +++ b/tests/darnit_reproducibility/test_implementation.py @@ -3,6 +3,14 @@ from darnit_reproducibility.implementation import ReproducibilityImplementation +def _framework_controls(impl): + """Controls as the audit loads them, from the implementation's framework TOML.""" + from darnit.config import load_controls_from_framework + from darnit.config.merger import load_framework_config + + return load_controls_from_framework(load_framework_config(impl.get_framework_config_path())) + + class TestReproducibilityImplementation: """Tests that ReproducibilityImplementation satisfies the plugin protocol.""" @@ -15,25 +23,16 @@ def test_name(self) -> None: def test_display_name(self) -> None: assert len(self.impl.display_name) > 0 - def test_get_all_controls_returns_five(self) -> None: - assert len(self.impl.get_all_controls()) == 5 + def test_framework_toml_defines_five_controls(self) -> None: + assert len(_framework_controls(self.impl)) == 5 def test_control_ids(self) -> None: - ids = {c.control_id for c in self.impl.get_all_controls()} + ids = {c.control_id for c in _framework_controls(self.impl)} assert ids == {"RE-01.01", "RE-01.02", "RE-02.01", "RE-02.02", "RE-03.01"} - def test_level_1_has_two_controls(self) -> None: - controls = self.impl.get_controls_by_level(1) - assert len(controls) == 2 - - def test_level_2_has_two_controls(self) -> None: - controls = self.impl.get_controls_by_level(2) - assert len(controls) == 2 - - def test_level_3_has_one_control(self) -> None: - controls = self.impl.get_controls_by_level(3) - assert len(controls) == 1 - assert controls[0].control_id == "RE-03.01" + def test_control_levels(self) -> None: + levels = {c.control_id: c.level for c in _framework_controls(self.impl)} + assert levels == {"RE-01.01": 1, "RE-01.02": 1, "RE-02.01": 2, "RE-02.02": 2, "RE-03.01": 3} def test_framework_config_path_exists(self) -> None: path = self.impl.get_framework_config_path() @@ -46,7 +45,7 @@ def test_check_handlers_covers_all_controls(self) -> None: reset_sieve_handler_registry, ) reset_sieve_handler_registry() - self.impl.register_sieve_handlers() + self.impl.register_handlers() registry = get_sieve_handler_registry() expected = { "repro_deps_pinned", "repro_build_env_declared", @@ -62,7 +61,7 @@ def test_all_check_handlers_are_callable(self) -> None: reset_sieve_handler_registry, ) reset_sieve_handler_registry() - self.impl.register_sieve_handlers() + self.impl.register_handlers() registry = get_sieve_handler_registry() for name in [ "repro_deps_pinned", "repro_build_env_declared", diff --git a/tests/darnit_reproducibility/test_no_pass_from_signals.py b/tests/darnit_reproducibility/test_no_pass_from_signals.py index 9cb54aa5..bee668a7 100644 --- a/tests/darnit_reproducibility/test_no_pass_from_signals.py +++ b/tests/darnit_reproducibility/test_no_pass_from_signals.py @@ -76,7 +76,7 @@ @pytest.fixture(autouse=True) def _registered() -> None: - ReproducibilityImplementation().register_sieve_handlers() + ReproducibilityImplementation().register_handlers() def _build(root: Path, files: dict[str, str]) -> Path: