diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 27491b2..65aeb78 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -47,4 +47,4 @@ jobs: run: uv sync --all-extras - name: Run pytest - run: uv run pytest --cov=mitreattack + run: uv run --extra dev pytest -n 2 --cov=mitreattack diff --git a/AGENTS.md b/AGENTS.md index ac820fb..5768d02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,9 @@ - Before committing, run `just lint`. - `just lint`: run pre-commit hooks across the repo. - `just test`: run the pytest suite. +- `just test-xdist`: run the pytest suite in parallel. - `just test-cov`: run tests with coverage for `mitreattack`. +- `just test-cov-xdist`: run tests with coverage in parallel. - `just build`: build distributions with `uv build`. - Without `just`, run the same tools through `uv run ...`. @@ -32,6 +34,10 @@ - Framework: `pytest` (with `pytest-cov` for coverage checks). - Place tests under `tests/` and name files/functions `test_*.py` / `test_*`. - Add or update tests for behavior changes, especially around STIX parsing and changelog/diff output paths. +- Tests that need real ATT&CK STIX data should use the shared STIX fixtures instead of downloading or + preparing bundles directly. +- Parallel runs warm the shared STIX cache before workers start; update `DEFAULT_ATTACK_STIX_PREP` in + `tests/conftest.py` if a new xdist-backed test needs another ATT&CK release. - Run `just test` locally before opening a PR; use `just test-cov` for larger changes. ## Commit & Pull Request Guidelines diff --git a/conftest.py b/conftest.py deleted file mode 100644 index f07a5f2..0000000 --- a/conftest.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Pytest command-line options for mitreattack-python.""" - - -def pytest_addoption(parser): - """Register pytest options for selecting ATT&CK STIX test data.""" - parser.addoption( - "--stix-enterprise", - action="store", - default=None, - help="Path to an Enterprise ATT&CK STIX bundle to use in tests.", - ) - parser.addoption( - "--stix-mobile", - action="store", - default=None, - help="Path to a Mobile ATT&CK STIX bundle to use in tests.", - ) - parser.addoption( - "--stix-ics", - action="store", - default=None, - help="Path to an ICS ATT&CK STIX bundle to use in tests.", - ) - parser.addoption( - "--attack-version", - action="store", - default=None, - help="ATT&CK release version to download and use for STIX-backed tests.", - ) - parser.addoption( - "--stix-version", - action="store", - choices=("2.0", "2.1"), - default="2.0", - help="STIX version to download when --attack-version is used. Defaults to 2.0.", - ) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index fda140f..131a0ad 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -52,10 +52,17 @@ Run `just` with no arguments to see all available commands. Here are the most co ```bash just lint # Run pre-commit hooks (ruff format) on all files just test # Run tests +just test-xdist # Run tests in parallel just test-cov # Run tests with coverage report +just test-cov-xdist # Run tests with coverage in parallel just build # Build the package ``` +Tests that need real ATT&CK STIX data should use the shared STIX fixtures instead of downloading or +preparing bundles directly. Parallel test runs warm the shared STIX cache before workers start; if a +new xdist-backed test needs an additional ATT&CK release, update the cache warmup list in +`tests/conftest.py`. + To run STIX-backed tests against specific local bundles, pass the bundle paths to pytest: ```bash diff --git a/justfile b/justfile index 85544c9..eb80af3 100644 --- a/justfile +++ b/justfile @@ -35,6 +35,14 @@ ruff-format: test: uv run pytest +# Run tests in parallel +test-xdist workers="auto": + uv run --extra dev pytest -n {{ workers }} + +# Run tests with coverage in parallel +test-cov-xdist workers="auto": + uv run --extra dev pytest -n {{ workers }} --cov=mitreattack + # Run tests with coverage test-cov: uv run pytest --cov=mitreattack diff --git a/pyproject.toml b/pyproject.toml index 0f56717..d67aceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "pytest>=8.4.2", "pytest-cov>=7.0.0", "pytest-dotenv>=0.5.2", + "pytest-xdist>=3.8.0", "python-semantic-release>=10.5.0", "responses>=0.25.8", "ruff>=0.14.2", @@ -131,3 +132,7 @@ version_files = [ "docs/conf.py:^release = ['\"](.*)['\"]", "mitreattack/__init__.py:^__version__ = ['\"](.*)['\"]", ] + +[[tool.uv.index]] +url = "https://pypi.org/simple" +default = true diff --git a/tests/changelog/conftest.py b/tests/changelog/conftest.py index 11344a4..d95e3e2 100644 --- a/tests/changelog/conftest.py +++ b/tests/changelog/conftest.py @@ -1594,10 +1594,10 @@ def golden_161_170_changelog_dir(): @pytest.fixture(scope="session") -def generated_161_170_diffstix(tmp_path_factory) -> DiffStix: +def generated_161_170_diffstix() -> DiffStix: """Create and cache a DiffStix instance for reuse across tests.""" versions_param = ["16.1", "17.0"] - result_paths = _download_attack_stix_data(versions_param, tmp_path_factory) + result_paths = _download_attack_stix_data(versions_param) return DiffStix( domains=["enterprise-attack", "mobile-attack", "ics-attack"], old=result_paths["16.1"], diff --git a/tests/conftest.py b/tests/conftest.py index 5e58f6c..d7249f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Common fixtures and utilities for testing mitreattack-python.""" import os +from pathlib import Path import pytest from loguru import logger @@ -23,6 +24,58 @@ "mobile": "stix_mobile", "ics": "stix_ics", } +TEST_CACHE_DIR = Path( + os.getenv("MITREATTACK_TEST_CACHE_DIR", Path(__file__).resolve().parent.parent / ".pytest_cache" / "attack-stix") +) +DEFAULT_ATTACK_STIX_PREP = (None, ["16.1", "17.0"]) + + +def pytest_addoption(parser): + """Register pytest options for selecting ATT&CK STIX test data.""" + parser.addoption( + "--stix-enterprise", + action="store", + default=None, + help="Path to an Enterprise ATT&CK STIX bundle to use in tests.", + ) + parser.addoption( + "--stix-mobile", + action="store", + default=None, + help="Path to a Mobile ATT&CK STIX bundle to use in tests.", + ) + parser.addoption( + "--stix-ics", + action="store", + default=None, + help="Path to an ICS ATT&CK STIX bundle to use in tests.", + ) + parser.addoption( + "--attack-version", + action="store", + default=None, + help="ATT&CK release version to download and use for STIX-backed tests.", + ) + parser.addoption( + "--stix-version", + action="store", + choices=("2.0", "2.1"), + default="2.0", + help="STIX version to download when --attack-version is used. Defaults to 2.0.", + ) + + +def pytest_sessionstart(session): + """Warm the ATT&CK STIX cache before xdist test sessions start.""" + config = session.config + if _should_prepare_attack_stix_cache(config) and not _all_stix_files_requested(config): + prep_params = list(DEFAULT_ATTACK_STIX_PREP) + requested_param = _get_requested_attack_stix_param(config) + if requested_param and not _is_attack_stix_param_prepared(requested_param, prep_params): + prep_params.append(requested_param) + + for versions_param in prep_params: + _download_attack_stix_data(versions_param, config=config) def _get_config_option(config, name): @@ -30,32 +83,48 @@ def _get_config_option(config, name): return config.getoption(name, default=None) -def _get_requested_stix_file(request, domain): +def _get_requested_stix_file(config, domain): """Get a configured STIX file path for a domain from CLI options or environment.""" - cli_value = _get_config_option(request.config, STIX_LOCATION_OPTIONS[domain]) + cli_value = _get_config_option(config, STIX_LOCATION_OPTIONS[domain]) if cli_value: return cli_value return os.getenv(STIX_LOCATION_ENV_VARS[domain]) -def _all_stix_files_requested(request): +def _all_stix_files_requested(config): """Return whether every domain has a configured local STIX file.""" - return all(_get_requested_stix_file(request, domain) for domain in STIX_LOCATION_OPTIONS) + return all(_get_requested_stix_file(config, domain) for domain in STIX_LOCATION_OPTIONS) -def _get_requested_attack_stix_param(request): +def _get_requested_attack_stix_param(config): """Build the attack_stix_dir fixture parameter from pytest version options.""" - attack_version = _get_config_option(request.config, "attack_version") + attack_version = _get_config_option(config, "attack_version") if not attack_version: return None return { "attack_version": attack_version, - "stix_version": _get_config_option(request.config, "stix_version") or "2.0", + "stix_version": _get_config_option(config, "stix_version") or "2.0", } +def _get_cache_request(versions_param): + """Return normalized cache request details for a version parameter.""" + versions, stix_version = _parse_version_param(versions_param) + return set(versions or [LATEST_VERSION]), stix_version + + +def _is_attack_stix_param_prepared(versions_param, prepared_params): + """Return whether a version parameter is already covered by prepared cache params.""" + requested_versions, requested_stix_version = _get_cache_request(versions_param) + for prepared_param in prepared_params: + prepared_versions, prepared_stix_version = _get_cache_request(prepared_param) + if requested_stix_version == prepared_stix_version and requested_versions.issubset(prepared_versions): + return True + return False + + def _parse_version_param(versions_param): """Parse version parameter into versions list and STIX version. @@ -119,7 +188,42 @@ def _get_stix_file_path(attack_stix_dir, domain, version_key="latest"): return f"{attack_stix_dir[first_version]}/{domain}-attack.json" -def _download_attack_stix_data(versions_param, tmp_path_factory): +def _get_release_dir(download_dir: Path, release: str) -> Path: + """Return the cache directory for a specific ATT&CK release.""" + return download_dir / f"v{release}" + + +def _is_xdist_worker(config=None) -> bool: + """Return whether the current process is an xdist worker.""" + if config is not None and getattr(config, "workerinput", None) is not None: + return True + return bool(os.getenv("PYTEST_XDIST_WORKER")) + + +def _should_prepare_attack_stix_cache(config=None) -> bool: + """Return whether the current pytest process should warm the shared STIX cache.""" + if _is_xdist_worker(config): + return False + + if config is None: + return False + + numprocesses = getattr(getattr(config, "option", None), "numprocesses", None) + return bool(numprocesses and int(numprocesses) > 0) + + +def _stix_bundles_present(download_dir: Path, releases: list[str]) -> bool: + """Check whether all cached STIX bundles needed for a run already exist.""" + domains = ["enterprise", "mobile", "ics"] + for release in releases: + release_dir = _get_release_dir(download_dir, release) + for domain in domains: + if not (release_dir / f"{domain}-attack.json").exists(): + return False + return True + + +def _download_attack_stix_data(versions_param, config=None): """Download ATT&CK STIX data and return paths. This is the core download logic shared by multiple fixtures. @@ -128,8 +232,8 @@ def _download_attack_stix_data(versions_param, tmp_path_factory): ---------- versions_param : None, str, list, or dict Version parameter to parse - tmp_path_factory : pytest.TempPathFactory - Pytest temp path factory + config : pytest.Config, optional + Active pytest config used to determine read-only cache behavior. Returns ------- @@ -137,32 +241,45 @@ def _download_attack_stix_data(versions_param, tmp_path_factory): Dictionary mapping version to download directory path """ versions, stix_version = _parse_version_param(versions_param) + requested_versions = versions or [LATEST_VERSION] - logger.debug(f"Downloading the ATT&CK STIX {stix_version} data for versions: {versions}") - download_dir = tmp_path_factory.mktemp("attack-releases") / f"stix-{stix_version}" + logger.debug(f"Preparing ATT&CK STIX {stix_version} data for versions: {requested_versions}") + download_dir = TEST_CACHE_DIR / f"stix-{stix_version}" + download_dir.mkdir(parents=True, exist_ok=True) - download_domains( - domains=["enterprise", "mobile", "ics"], - download_dir=download_dir, - all_versions=False, - stix_version=stix_version, - attack_versions=versions, - ) + if _stix_bundles_present(download_dir, requested_versions): + logger.debug(f"Reusing cached ATT&CK STIX bundles from {download_dir}") + else: + if _is_xdist_worker(config): + requested = ", ".join(requested_versions) + raise RuntimeError( + "ATT&CK STIX cache is missing required bundles for " + f"{requested}. xdist runs should warm this cache before workers start. " + "If you added a new ATT&CK version to xdist-backed tests, update DEFAULT_ATTACK_STIX_PREP." + ) + logger.debug(f"Downloading ATT&CK STIX bundles into cache at {download_dir}") + download_domains( + domains=["enterprise", "mobile", "ics"], + download_dir=download_dir, + all_versions=False, + stix_version=stix_version, + attack_versions=versions, + ) # Build return dictionary result_paths = {} if versions is None: - result_paths["latest"] = download_dir / f"v{LATEST_VERSION}" + result_paths["latest"] = _get_release_dir(download_dir, LATEST_VERSION) else: # Return paths for each requested version for version in versions: - result_paths[version] = download_dir / f"v{version}" + result_paths[version] = _get_release_dir(download_dir, version) return result_paths -@pytest.fixture(autouse=True, scope="session") -def attack_stix_dir(request, tmp_path_factory): +@pytest.fixture(scope="session") +def attack_stix_dir(request): """Download ATT&CK STIX data and return paths. Can be parametrized to download specific versions: @@ -188,12 +305,12 @@ def attack_stix_dir(request, tmp_path_factory): dict Directory paths for requested ATT&CK versions """ - if _all_stix_files_requested(request): + if _all_stix_files_requested(request.config): yield {} return - versions_param = getattr(request, "param", None) or _get_requested_attack_stix_param(request) - result_paths = _download_attack_stix_data(versions_param, tmp_path_factory) + versions_param = getattr(request, "param", None) or _get_requested_attack_stix_param(request.config) + result_paths = _download_attack_stix_data(versions_param, config=request.config) yield result_paths @@ -213,7 +330,7 @@ def stix_file_enterprise_latest(request, attack_stix_dir): str Path to Enterprise ATT&CK STIX file """ - requested_stix_file = _get_requested_stix_file(request, "enterprise") + requested_stix_file = _get_requested_stix_file(request.config, "enterprise") if requested_stix_file: return requested_stix_file @@ -236,7 +353,7 @@ def stix_file_mobile_latest(request, attack_stix_dir): str Path to Mobile ATT&CK STIX file """ - requested_stix_file = _get_requested_stix_file(request, "mobile") + requested_stix_file = _get_requested_stix_file(request.config, "mobile") if requested_stix_file: return requested_stix_file @@ -259,7 +376,7 @@ def stix_file_ics_latest(request, attack_stix_dir): str Path to ICS ATT&CK STIX file """ - requested_stix_file = _get_requested_stix_file(request, "ics") + requested_stix_file = _get_requested_stix_file(request.config, "ics") if requested_stix_file: return requested_stix_file diff --git a/uv.lock b/uv.lock index 22ced1c..013bd72 100644 --- a/uv.lock +++ b/uv.lock @@ -429,6 +429,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "filelock" version = "3.24.3" @@ -674,6 +683,7 @@ dev = [ { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-dotenv" }, + { name = "pytest-xdist" }, { name = "python-semantic-release" }, { name = "responses" }, { name = "ruff" }, @@ -702,6 +712,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.2" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "pytest-dotenv", marker = "extra == 'dev'", specifier = ">=0.5.2" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "python-dateutil", specifier = ">=2.8.2" }, { name = "python-semantic-release", marker = "extra == 'dev'", specifier = ">=10.5.0" }, { name = "requests", specifier = ">=2.31.0" }, @@ -1210,6 +1221,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/da/9da67c67b3d0963160e3d2cbc7c38b6fae342670cc8e6d5936644b2cf944/pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f", size = 3993, upload-time = "2020-06-16T12:38:01.139Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"