diff --git a/.github/workflows/wc-document-generation.yml b/.github/workflows/wc-document-generation.yml index 6088e47e..abeb6f1b 100644 --- a/.github/workflows/wc-document-generation.yml +++ b/.github/workflows/wc-document-generation.yml @@ -23,32 +23,48 @@ jobs: run: | set -Eeuo pipefail sudo apt-get update && sudo apt-get install --no-install-recommends -y plantuml - python -m pip install gherkin-official==35.1.0 sbdl==1.16.4 + python -m pip install sbdl==1.16.4 - name: Generate SRS document run: | set -Eeuo pipefail - python docs/support/generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature - sbdl -m template-fill --template docs/templates/software-requirements-specification.md.j2 output.sbdl > software-requirements-specification.md + sbdl -m template-fill \ + --template docs/templates/software-requirements-specification.md.j2 \ + test/cpp/features/*.feature \ + > software-requirements-specification.md - name: Generate C++ test plan and traceability documents run: | set -Eeuo pipefail - python docs/support/generate-sbdl.py --config test-specification \ - --gherkin test/cpp/features/*.feature \ - --bats test/cpp/integration-tests.bats test/base/integration-tests.bats \ - --flavor "C++ (cpp)" \ - -o cpp-test-plan.sbdl - sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 cpp-test-plan.sbdl > cpp-software-test-plan.md - sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 cpp-test-plan.sbdl > cpp-requirements-traceability-matrix.md + sbdl -m template-fill \ + --template docs/templates/software-test-plan.md.j2 \ + docs/support/cpp-flavor.sbdl \ + test/cpp/features/*.feature \ + test/cpp/integration-tests.bats \ + test/base/integration-tests.bats \ + > cpp-software-test-plan.md + sbdl -m template-fill \ + --template docs/templates/requirements-traceability-matrix.md.j2 \ + docs/support/cpp-flavor.sbdl \ + test/cpp/features/*.feature \ + test/cpp/integration-tests.bats \ + test/base/integration-tests.bats \ + > cpp-requirements-traceability-matrix.md - name: Generate Rust test plan and traceability documents run: | set -Eeuo pipefail - python docs/support/generate-sbdl.py --config test-specification \ - --gherkin test/cpp/features/*.feature \ - --bats test/rust/integration-tests.bats test/base/integration-tests.bats \ - --flavor Rust \ - -o rust-test-plan.sbdl - sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 rust-test-plan.sbdl > rust-software-test-plan.md - sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 rust-test-plan.sbdl > rust-requirements-traceability-matrix.md + sbdl -m template-fill \ + --template docs/templates/software-test-plan.md.j2 \ + docs/support/rust-flavor.sbdl \ + test/cpp/features/*.feature \ + test/rust/integration-tests.bats \ + test/base/integration-tests.bats \ + > rust-software-test-plan.md + sbdl -m template-fill \ + --template docs/templates/requirements-traceability-matrix.md.j2 \ + docs/support/rust-flavor.sbdl \ + test/cpp/features/*.feature \ + test/rust/integration-tests.bats \ + test/base/integration-tests.bats \ + > rust-requirements-traceability-matrix.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: args: >- diff --git a/docs/support/bats_sbdl_converter.py b/docs/support/bats_sbdl_converter.py deleted file mode 100644 index 448e1399..00000000 --- a/docs/support/bats_sbdl_converter.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""BATS test file to SBDL converter. - -Parses BATS test files and extracts @test definitions along with their -optional `# bats test_tags=` annotations, producing SBDL `test` elements -with traceability to requirements via tag-based mapping. -""" - -import re -from dataclasses import dataclass, field -from typing import Dict, List, Optional - -from gherkin_sbdl_converter import to_slug - -try: - import sbdl - - def sanitize_description(text: str) -> str: - return sbdl.SBDL_Parser.sanitize(text) - -except ImportError: - def sanitize_description(text: str) -> str: - return text.replace('"', '\\"') - - -@dataclass -class BatsTest: - """Represents a single BATS test case.""" - - name: str - identifier: str - tags: List[str] = field(default_factory=list) - file_path: str = "" - line_number: int = 0 - - def __post_init__(self): - if not self.identifier: - self.identifier = to_slug(self.name) - - -class BatsConverter: - """Converts BATS test files to SBDL test elements.""" - - # Regex patterns for BATS file parsing - _TAG_PATTERN = re.compile(r"^#\s*bats\s+test_tags\s*=\s*(.+)$", re.IGNORECASE) - _TEST_PATTERN = re.compile(r'^@test\s+"(.+?)"\s*\{', re.MULTILINE) - - def __init__(self, requirement_tag_map: Optional[Dict[str, str]] = None): - """Initialize the converter. - - Args: - requirement_tag_map: Optional mapping from tag names to requirement - identifiers (SBDL element IDs). When provided, tests with - matching tags will get a `requirement` relation in SBDL output. - """ - self.requirement_tag_map = requirement_tag_map or {} - - def extract_from_bats_file(self, file_path: str, flavor: str = "") -> List[BatsTest]: - """Extract all test definitions from a BATS file. - - Parses `@test "..."` blocks and any preceding `# bats test_tags=...` - comment lines. - - Args: - file_path: Path to the .bats file. - flavor: Optional identifier prefix for disambiguation - (e.g. 'cpp', 'rust', 'base'). - - Returns: - List of BatsTest objects. - """ - try: - with open(file_path, "r", encoding="utf-8") as f: - lines = f.readlines() - except (IOError, OSError) as e: - print(f"Error reading {file_path}: {e}") - return [] - - tests: List[BatsTest] = [] - pending_tags: List[str] = [] - - for line_number, line in enumerate(lines, start=1): - stripped = line.strip() - - # Check for tag annotation - tag_match = self._TAG_PATTERN.match(stripped) - if tag_match: - tags_str = tag_match.group(1) - pending_tags = [t.strip() for t in tags_str.split(",") if t.strip()] - continue - - # Check for test definition - test_match = self._TEST_PATTERN.match(stripped) - if test_match: - test_name = test_match.group(1) - base_id = to_slug(test_name) - identifier = f"{flavor}-{base_id}" if flavor else base_id - test = BatsTest( - name=test_name, - identifier=identifier, - tags=list(pending_tags), - file_path=file_path, - line_number=line_number, - ) - tests.append(test) - pending_tags = [] - print(f" Extracted BATS test: {test.identifier}") - continue - - # Reset pending tags if we hit a non-comment, non-empty line - # that isn't a test (so tags don't bleed across unrelated lines) - if stripped and not stripped.startswith("#"): - pending_tags = [] - - return tests - - def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: - """Resolve typed relations from test tags using the tag map. - - Groups resolved identifiers by their SBDL element type, so the - correct relation keyword (e.g. 'aspect', 'requirement') is used - in the SBDL output. - - Args: - test: A BatsTest with tags. - - Returns: - Dict mapping SBDL type names to lists of identifiers. - """ - relations: Dict[str, List[str]] = {} - for tag in test.tags: - tag_lower = tag.lower() - if tag_lower in self.requirement_tag_map: - entry = self.requirement_tag_map[tag_lower] - if isinstance(entry, tuple): - identifier, elem_type = entry - else: - identifier, elem_type = entry, "requirement" - if elem_type not in relations: - relations[elem_type] = [] - if identifier not in relations[elem_type]: - relations[elem_type].append(identifier) - return relations diff --git a/docs/support/cpp-flavor.sbdl b/docs/support/cpp-flavor.sbdl new file mode 100644 index 00000000..1dbc0c63 --- /dev/null +++ b/docs/support/cpp-flavor.sbdl @@ -0,0 +1,2 @@ +#!sbdl +document-flavor is definition { description is "C++ (cpp)" custom:title is "C++ (cpp)" } diff --git a/docs/support/generate-sbdl.py b/docs/support/generate-sbdl.py deleted file mode 100644 index 1b693b6f..00000000 --- a/docs/support/generate-sbdl.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -"""Unified converter for generating SBDL from Gherkin feature files and BATS test files. - -Supports multiple output configurations: - - requirements: Feature/Rule hierarchy as requirements (for SRS) - - test-specification: Feature→aspect, Rule→requirement, Scenario→test + BATS→test (for test spec & traceability) - -Usage examples: - # Generate requirements SBDL (existing behavior) - python generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature - - # Generate test specification SBDL with both Gherkin scenarios and BATS tests - python generate-sbdl.py --config test-specification \\ - --gherkin test/cpp/features/*.feature \\ - --bats test/cpp/integration-tests.bats test/base/integration-tests.bats -""" - -import argparse -import os -import sys - -from gherkin_mapping_config import FEATURE_RULE_CONFIG, TEST_SPECIFICATION_CONFIG -from gherkin_sbdl_converter import GherkinConverter, to_slug, write_gherkin_sbdl_elements -from bats_sbdl_converter import BatsConverter - - -def main(): - configs = { - "requirements": FEATURE_RULE_CONFIG, - "test-specification": TEST_SPECIFICATION_CONFIG, - } - - parser = argparse.ArgumentParser( - description="Unified Gherkin + BATS to SBDL converter" - ) - parser.add_argument( - "--gherkin", - nargs="*", - default=[], - help="Paths to Gherkin feature files", - ) - parser.add_argument( - "--bats", - nargs="*", - default=[], - help="Paths to BATS test files", - ) - parser.add_argument( - "--output", - "-o", - default="output.sbdl", - help="Output SBDL file", - ) - parser.add_argument( - "--config", - choices=configs.keys(), - default="requirements", - help="Conversion configuration preset", - ) - parser.add_argument( - "--flavor", - default="", - help="Container flavor name for per-flavor document generation", - ) - - args = parser.parse_args() - - if not args.gherkin and not args.bats: - parser.error("At least one --gherkin or --bats file must be specified") - - config = configs[args.config] - gherkin_converter = GherkinConverter(config) - gherkin_elements = [] - - # Process Gherkin feature files - for feature_path in args.gherkin: - if os.path.isfile(feature_path): - print(f"Processing Gherkin: {feature_path}") - elements = gherkin_converter.extract_from_feature_file(feature_path) - gherkin_elements.extend(elements) - else: - print(f"File not found: {feature_path}", file=sys.stderr) - - # Build requirement tag map from Gherkin elements for BATS traceability. - # Maps lowercased sanitized identifiers and feature names to (identifier, element_type) - # tuples, enabling BATS tests tagged with e.g. "Compatibility" to trace to the - # corresponding Gherkin element with the correct SBDL relation type. - requirement_tag_map = {} - for elem in gherkin_elements: - if elem.metadata and elem.metadata.get("gherkin_type") in ("feature", "rule"): - entry = (elem.identifier, elem.element_type.value) - original = elem.metadata.get("original_name", "") - # Map the original feature/rule name (lowercased) to the SBDL identifier + type - if original: - requirement_tag_map[original.lower()] = entry - # Also map the slugified version for BATS tag matching - slug_original = to_slug(original) - if slug_original: - requirement_tag_map[slug_original] = entry - # Also map the SBDL identifier itself - requirement_tag_map[elem.identifier.lower()] = entry - - # Process BATS test files - bats_converter = BatsConverter(requirement_tag_map=requirement_tag_map) - bats_tests = [] - - for bats_path in args.bats: - if os.path.isfile(bats_path): - print(f"Processing BATS: {bats_path}") - flavor_prefix = to_slug(os.path.basename(os.path.dirname(os.path.abspath(bats_path)))) - tests = bats_converter.extract_from_bats_file(bats_path, flavor=flavor_prefix) - bats_tests.extend(tests) - else: - print(f"File not found: {bats_path}", file=sys.stderr) - - # Write combined SBDL output - _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, args.output, args.flavor) - - total = len(gherkin_elements) + len(bats_tests) - print(f"Extracted {total} elements ({len(gherkin_elements)} from Gherkin, {len(bats_tests)} from BATS) to {args.output}") - - -def _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, output_file, flavor=""): - """Write a single combined SBDL file from both Gherkin and BATS sources.""" - import sbdl as sbdl_lib - - with open(output_file, "w", encoding="utf-8") as f: - f.write("#!sbdl\n\n") - - # Write flavor marker element for per-flavor document generation - if flavor: - escaped_flavor = sbdl_lib.SBDL_Parser.sanitize(flavor) - f.write(f'document-flavor is definition {{ description is "{escaped_flavor}" custom:title is "{escaped_flavor}" }}\n\n') - - f.write("# Elements extracted from Gherkin feature files\n") - write_gherkin_sbdl_elements(f, gherkin_elements) - - if bats_tests: - f.write("\n# Elements extracted from BATS integration test files\n") - - for test in bats_tests: - escaped_desc = sbdl_lib.SBDL_Parser.sanitize(test.name) - identifier = test.identifier - - f.write(f'{identifier} is test {{ description is "{escaped_desc}" custom:title is "{escaped_desc}"') - - if test.tags: - tag_str = ",".join(test.tags) - f.write(f" tag is {tag_str}") - - resolved = bats_converter._resolve_typed_relations(test) - for rel_type, rel_ids in resolved.items(): - f.write(f" {rel_type} is {','.join(rel_ids)}") - - f.write(" }\n") - - -if __name__ == "__main__": - main() diff --git a/docs/support/gherkin_mapping_config.py b/docs/support/gherkin_mapping_config.py deleted file mode 100644 index fde6ee1f..00000000 --- a/docs/support/gherkin_mapping_config.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -""" -Configuration-driven Gherkin to SBDL mapping with flexible hierarchy support. -""" - -from dataclasses import dataclass -from typing import List, Optional -from enum import Enum - -class GherkinElementType(Enum): - """Supported Gherkin element types for conversion.""" - FEATURE = "feature" - RULE = "rule" - SCENARIO = "scenario" - SCENARIO_OUTLINE = "scenario_outline" - EXAMPLE = "example" - BACKGROUND = "background" - -class SBDLElementType(Enum): - """Supported SBDL element types for mapping.""" - REQUIREMENT = "requirement" - ASPECT = "aspect" - TEST = "test" - -@dataclass -class HierarchyMapping: - """Configuration for mapping Gherkin elements to SBDL with document hierarchy.""" - gherkin_type: GherkinElementType - sbdl_type: SBDLElementType - -@dataclass -class ConversionConfig: - """Complete configuration for Gherkin to SBDL conversion.""" - hierarchy_mappings: List[HierarchyMapping] - - def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[HierarchyMapping]: - """Get the hierarchy mapping for a specific Gherkin element type.""" - for mapping in self.hierarchy_mappings: - if mapping.gherkin_type == gherkin_type: - return mapping - return None - -# Predefined configurations for different use cases - -# Requirements Configuration (Feature -> Rule mapping as requirements) -FEATURE_RULE_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.REQUIREMENT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.RULE, - sbdl_type=SBDLElementType.REQUIREMENT - ) - ] -) - -# Test Specification Configuration (Feature -> Rule -> Scenario with tests) -# Used for generating test specification and traceability documents. -# Features become aspects (system areas), Rules become requirements, -# and Scenarios become test elements traced to their parent requirements. -TEST_SPECIFICATION_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.ASPECT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.RULE, - sbdl_type=SBDLElementType.REQUIREMENT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.SCENARIO, - sbdl_type=SBDLElementType.TEST - ) - ] -) diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py deleted file mode 100644 index 5787d1f7..00000000 --- a/docs/support/gherkin_sbdl_converter.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -"""Gherkin to SBDL converter with configurable hierarchy mappings.""" - -from dataclasses import dataclass -from typing import List, Dict, Optional -from textwrap import dedent -import re -import sys - -from gherkin.parser import Parser -from gherkin.token_scanner import TokenScanner -import sbdl - -from gherkin_mapping_config import ( - ConversionConfig, - GherkinElementType, - SBDLElementType, -) - - -def to_slug(name: str) -> str: - """Convert a human-readable name to a hyphenated slug identifier. - - Lowercases the name and replaces any sequence of non-alphanumeric - characters with a single hyphen. - - Args: - name: The original name. - - Returns: - Hyphenated, lowercase slug suitable as an SBDL identifier. - """ - slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) - return slug.strip('-') - -@dataclass -class SBDLElement: - """A generalized SBDL element that can represent any type.""" - - identifier: str - element_type: SBDLElementType - description: str - parent: Optional[str] = None - metadata: Optional[Dict] = None - - def __post_init__(self): - self.identifier = self._make_sbdl_identifier(self.identifier) - self.description = dedent(self.description) - - if self.metadata is None: - self.metadata = {} - - def _make_sbdl_identifier(self, name: str) -> str: - """Convert a name to a hyphenated slug identifier.""" - return to_slug(name) - -def write_gherkin_sbdl_elements(f, elements: List[SBDLElement]): - """Write a list of Gherkin-derived SBDL elements to an open file handle. - - Handles description, custom:title, and parent/cross-type relations - for each element. - - Args: - f: Writable file handle. - elements: List of SBDLElement objects to write. - """ - tokens = sbdl.SBDL_Parser.Tokens - attrs = sbdl.SBDL_Parser.Attributes - - # Build a lookup from identifier to element type for cross-type relation handling - element_types = {e.identifier: e.element_type for e in elements} - - for element in elements: - escaped_desc = sbdl.SBDL_Parser.sanitize(element.description) - sbdl_type = element.element_type.value - f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") - f.write(f"{tokens.declaration_group_delimeters[0]} ") - f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") - f.write( - f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " - ) - - # Write custom:title with original name for display purposes - original_name = element.metadata.get('original_name', '') if element.metadata else '' - if original_name: - escaped_title = sbdl.SBDL_Parser.sanitize(original_name) - f.write(f'custom:title{tokens.declaration_attribute_assign}') - f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') - - if element.parent: - parent_type = element_types.get(element.parent) - if parent_type and parent_type != element.element_type: - # Cross-type relation: use the parent's type name as relation - # e.g. test -> requirement becomes "requirement is " - # e.g. requirement -> aspect becomes "aspect is " - f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") - else: - f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") - - f.write(f"{tokens.declaration_group_delimeters[1]}\n") - - -class GherkinConverter: - """Converts Gherkin files to SBDL using configurable hierarchy mappings.""" - - def __init__(self, config: ConversionConfig): - self.config = config - # Strategy map for element extraction: key in parsed Gherkin dict -> handler. - self._strategies = { - 'feature': self._handle_feature, - 'rule': self._handle_rule, - 'scenario': self._handle_scenario, - 'scenarioOutline': self._handle_scenario_outline, - } - - def extract_from_feature_file(self, file_path: str) -> List[SBDLElement]: - """Extract all configured elements from a Gherkin feature file.""" - try: - with open(file_path, 'r', encoding='utf-8') as file: - content = file.read() - - parser = Parser() - feature_document = parser.parse(TokenScanner(content)) - - if 'feature' not in feature_document: - return [] - - return self._dispatch(feature_document, None) - except Exception as e: - print(f"Error parsing {file_path}: {e}", file=sys.stderr) - return [] - - def write_sbdl_output(self, elements: List[SBDLElement], output_file: str): - """Write extracted Gherkin elements as a standalone SBDL file.""" - with open(output_file, 'w', encoding='utf-8') as f: - f.write("#!sbdl\n") - write_gherkin_sbdl_elements(f, elements) - - def _extract_gherkin_element(self, element_data: Dict, element_type: GherkinElementType, parent_id: Optional[str]) -> Optional[SBDLElement]: - mapping = self.config.get_mapping_for_type(element_type) - - if not mapping: - return None - - name = element_data.get('name', '').strip() - - if not name: - return None - - return SBDLElement( - identifier=name, - element_type=mapping.sbdl_type, - description=element_data.get('description', ''), - parent=parent_id, - metadata={ - 'gherkin_type': element_type.value, - 'original_name': name, - 'line': element_data.get('location', {}).get('line'), - }, - ) - - def _dispatch(self, node: Dict, parent_id: Optional[str]) -> List[SBDLElement]: - for key, handler in self._strategies.items(): - if key in node: - return handler(node[key], parent_id) - return [] - - def _handle_feature(self, feature_data: Dict, parent_id: Optional[str]) -> List[SBDLElement]: - elements: List[SBDLElement] = [] - feature_element = self._extract_gherkin_element(feature_data, GherkinElementType.FEATURE, parent_id) - feature_id = None - if feature_element: - elements.append(feature_element) - feature_id = feature_element.identifier - print(f"Extracted Feature: {feature_element.identifier}") - for child in feature_data.get('children', []): - elements.extend(self._dispatch(child, feature_id)) - return elements - - def _handle_rule(self, rule_data: Dict, parent_id: Optional[str]) -> List[SBDLElement]: - elements: List[SBDLElement] = [] - rule_element = self._extract_gherkin_element(rule_data, GherkinElementType.RULE, parent_id) - if not rule_element: - return elements - elements.append(rule_element) - print(f" Extracted Rule: {rule_element.identifier}") - for rule_child in rule_data.get('children', []): - elements.extend(self._dispatch(rule_child, rule_element.identifier)) - return elements - - def _handle_scenario(self, scenario_data: Dict, parent_id: Optional[str]) -> List[SBDLElement]: - scenario_element = self._extract_gherkin_element(scenario_data, GherkinElementType.SCENARIO, parent_id) - if not scenario_element: - return [] - print(f" Extracted Scenario: {scenario_element.identifier}") - return [scenario_element] - - def _handle_scenario_outline(self, outline_data: Dict, parent_id: Optional[str]) -> List[SBDLElement]: - outline_element = self._extract_gherkin_element(outline_data, GherkinElementType.SCENARIO_OUTLINE, parent_id) - if not outline_element: - return [] - print(f" Extracted Scenario Outline: {outline_element.identifier}") - return [outline_element] diff --git a/docs/support/rust-flavor.sbdl b/docs/support/rust-flavor.sbdl new file mode 100644 index 00000000..9d56dfed --- /dev/null +++ b/docs/support/rust-flavor.sbdl @@ -0,0 +1,2 @@ +#!sbdl +document-flavor is definition { description is "Rust" custom:title is "Rust" } diff --git a/test/base/integration-tests.bats b/test/base/integration-tests.bats index 0d482420..0b3d9184 100644 --- a/test/base/integration-tests.bats +++ b/test/base/integration-tests.bats @@ -8,6 +8,7 @@ setup() { } # bats test_tags=security +# @sbdl base-cisco-umbrella-cert is test { aspect is security } @test "cisco umbrella root certificate is included in system certificate store" { run openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/cisco-umbrella-root.pem assert_success diff --git a/test/cpp/features/compatibility.feature b/test/cpp/features/compatibility.feature index 65cdebd8..d803d3ff 100644 --- a/test/cpp/features/compatibility.feature +++ b/test/cpp/features/compatibility.feature @@ -1,15 +1,18 @@ Feature: Compatibility + # @sbdl compatibility is aspect { description is "As a software craftsperson, to ensure that my development environment works well with a variety of tools and systems, I want my development environment to be compatible with commonly used tools and systems." } As a software craftsperson, to ensure that my development environment works well with a variety of tools and systems, I want my development environment to be compatible with commonly used tools and systems. Rule: Open Container Initiative (OCI) Image Specification + # @sbdl open-container-initiative-oci-image-specification is requirement { aspect is compatibility description is "amp-devcontainer images SHALL be compatible with the OCI image specification." } amp-devcontainer images *SHALL* be compatible with the [OCI image specification](https://github.com/opencontainers/image-spec/blob/main/spec.md) To guarantee compatibility with container runtimes and container- and image tooling; amp-devcontainer should be compatible with the OCI image specification. Rule: Host architecture + # @sbdl host-architecture is requirement { aspect is compatibility description is "amp-devcontainer SHALL be compatible with both the x86-64 (AMD64) and AArch64 (ARM64) host architectures." } amp-devcontainer *SHALL* be compatible with both the x86-64 (AMD64) *and* AArch64 (ARM64) host architectures. Supporting both x86-64 and AArch64 has several advantages: @@ -18,6 +21,7 @@ Feature: Compatibility - Unlocking the power efficiency of the AArch64 architecture, potentially reducing energy consumption and cost for metered ci-systems Rule: Integrated Development Environment (IDE) + # @sbdl integrated-development-environment-ide is requirement { aspect is compatibility description is "amp-devcontainer SHOULD be compatible with VS Code and GitHub Codespaces." } amp-devcontainer *SHOULD* be compatible with [VS Code](https://code.visualstudio.com/) *and* [GitHub Codespaces](https://github.com/features/codespaces). It should be possible to use amp-devcontainer and all of its features in both VS Code and GitHub Codespaces with minimal effort. @@ -25,6 +29,7 @@ Feature: Compatibility Features and functions should work "out-of-the-box" without being overly opinionated. Rule: GitHub Actions + # @sbdl github-actions is requirement { aspect is compatibility description is "amp-devcontainer SHOULD support seamless integration with GitHub Actions without additional configuration." } amp-devcontainer *SHOULD* support seamless integration with [GitHub Actions](https://github.com/features/actions) without additional configuration. Seamless integration with GitHub Actions allows users to easily incorporate amp-devcontainer into their ci/cd workflows. diff --git a/test/cpp/features/compilation.feature b/test/cpp/features/compilation.feature index de0e3e3c..c19401c3 100644 --- a/test/cpp/features/compilation.feature +++ b/test/cpp/features/compilation.feature @@ -1,10 +1,12 @@ Feature: Compilation + # @sbdl compilation is aspect { description is "As a software developer, to generate a working product, when using compiled languages, source code needs to be compiled into working software." } As a software developer, to generate a working product, when using compiled languages, source code needs to be compiled into working software. Rule: Compile for container host architecture and operating system + # @sbdl compile-for-container-host-architecture-and-operating-system is requirement { aspect is compilation description is "amp-devcontainer SHALL be able to compile valid source code into a working executable targeting the container host architecture and operating system." } amp-devcontainer *SHALL* be able to compile valid source code into a working executable targeting the container host architecture and operating system. Compiling valid source code into working software, able to run on the container host architecture and operating system, @@ -16,12 +18,14 @@ Feature: Compilation @flavor:cpp Scenario: Compile valid source code into working software targeting the container host architecture + # @sbdl compile-host-cpp is test { requirement is compile-for-container-host-architecture-and-operating-system } Given build configuration "gcc" is selected And build preset "gcc" is selected When the selected target is built Then the output should contain "Build finished with exit code 0" Rule: Compile for ARM Cortex target architecture + # @sbdl compile-for-arm-cortex-target-architecture is requirement { aspect is compilation description is "amp-devcontainer SHOULD be able to compile valid source-code into a working ELF executable targeting the ARM Cortex architecture." } amp-devcontainer *SHOULD* be able to compile valid source-code into a working ELF executable targeting the ARM Cortex architecture. Compiling valid source-code into working ELF executables, able to run on the ARM Cortex architecture, @@ -29,6 +33,7 @@ Feature: Compilation on the ARM Cortex architecture. Rule: Compile for Microsoft® Windows operating system + # @sbdl compile-for-microsoft-windows-operating-system is requirement { aspect is compilation description is "amp-devcontainer SHOULD be able to compile valid source-code into a working executable targeting the Microsoft® Windows operating system." } amp-devcontainer *SHOULD* be able to compile valid source-code into a working executable targeting the Microsoft® Windows operating system. Compiling valid source-code into working executables, able to run on the Microsoft® Windows operating system, can be necessary in several scenarios e.g. @@ -37,6 +42,7 @@ Feature: Compilation - Executables needs to be deployed outside of container context to a host running the Microsoft® Windows operating system Rule: Compilation cache + # @sbdl compilation-cache is requirement { aspect is compilation description is "amp-devcontainer MAY be able to cache the results of a compilation to speed-up subsequent compilations." } amp-devcontainer *MAY* be able to cache the results of a compilation to speed-up subsequent compilations. Maintaining a compilation cache can be useful in both local and ci development scenarios. A compilation cache can provide benefits like: diff --git a/test/cpp/features/debugging.feature b/test/cpp/features/debugging.feature index 58a64287..f0831e2d 100644 --- a/test/cpp/features/debugging.feature +++ b/test/cpp/features/debugging.feature @@ -1,10 +1,12 @@ Feature: Debugging + # @sbdl debugging is aspect { description is "As a software craftsperson, to efficiently identify and resolve issues in my code, I want to be able to debug my source code within the development environment." } As a software craftsperson, to efficiently identify and resolve issues in my code, I want to be able to debug my source code within the development environment. Rule: Debugging support + # @sbdl debugging-support is requirement { aspect is debugging description is "amp-devcontainer SHALL provide debugging support for the primary programming language(s) used within the container." } amp-devcontainer *SHALL* provide debugging support for the primary programming language(s) used within the container. Providing debugging support within the development environment enhances the developer experience and productivity. @@ -13,6 +15,7 @@ Feature: Debugging By having integrated debugging tools, developers can streamline their workflow and reduce the time spent on troubleshooting and fixing bugs. Rule: Upload firmware to micro-controller + # @sbdl upload-firmware-to-micro-controller is requirement { aspect is debugging description is "amp-devcontainer MAY provide tools to upload compiled firmware to a connected micro-controller." } amp-devcontainer *MAY* provide tools to upload compiled firmware to a connected micro-controller. Providing tools to upload compiled firmware to a connected micro-controller enhances the development workflow for embedded systems. diff --git a/test/cpp/features/maintainability.feature b/test/cpp/features/maintainability.feature index aa384582..fe8fa50b 100644 --- a/test/cpp/features/maintainability.feature +++ b/test/cpp/features/maintainability.feature @@ -1,10 +1,12 @@ Feature: Maintainability + # @sbdl maintainability is aspect { description is "As a software craftsperson, to ensure that I have access to a stable and reliable development environment, I want my development environment to be maintainable over time." } As a software craftsperson, to ensure that I have access to a stable and reliable development environment, I want my development environment to be maintainable over time. Rule: Tool and dependency updates + # @sbdl tool-and-dependency-updates is requirement { aspect is maintainability description is "amp-devcontainer SHOULD contain up-to-date tools and dependencies." } amp-devcontainer *SHOULD* contain up-to-date tools and dependencies. Keeping tools and dependencies up-to-date helps ensure that the development environment remains secure, stable, and compatible with the latest technologies. @@ -12,6 +14,7 @@ Feature: Maintainability Regular updates can also introduce new features and improvements that enhance the development experience. Rule: Automatic updates + # @sbdl automatic-updates is requirement { aspect is maintainability description is "amp-devcontainer SHOULD provide support for automatic updates when consumed as a dependency." } amp-devcontainer *SHOULD* provide support for automatic updates when consumed as a dependency. Providing support for automatic updates when amp-devcontainer is consumed as a dependency helps ensure that users always have access to the latest features, bug fixes, and security patches. @@ -19,12 +22,14 @@ Feature: Maintainability Automatic updates can also help ensure compatibility with other dependencies and tools, improving the overall stability and reliability of the development environment. Rule: Re-usable build system + # @sbdl re-usable-build-system is requirement { aspect is maintainability description is "amp-devcontainer SHOULD provide re-usable building blocks to enable building, publishing and testing derived containers." } amp-devcontainer *SHOULD* provide re-usable building blocks to enable building, publishing and testing derived containers. Providing re-usable building blocks for building, publishing and testing derived containers reduces duplication, and ensures consistent application of practices. Derived containers (i.e. containers using amp-devcontainer as a base for further extension) should be able to build, push and test in the same way that amp-devcontainer does, without the need to duplicate the build system. Rule: Architectural decisions + # @sbdl architectural-decisions is requirement { aspect is maintainability description is "amp-devcontainer SHOULD document its architectural decisions." } amp-devcontainer *SHOULD* document its architectural decisions. Documenting architectural decisions helps provide context and rationale for the design choices made in the development environment. @@ -32,6 +37,7 @@ Feature: Maintainability Clear documentation of architectural decisions can also facilitate collaboration among team members and improve overall maintainability. Rule: Container image size + # @sbdl container-image-size is requirement { aspect is maintainability description is "amp-devcontainer SHOULD aim to keep its container image size as small as possible without compromising functionality." } amp-devcontainer *SHOULD* aim to keep its container image size as small as possible without compromising functionality. Keeping the container image size small helps improve performance, reduce resource consumption, and minimize the time required to download and deploy the development environment. diff --git a/test/cpp/features/security.feature b/test/cpp/features/security.feature index 061a0053..e3266ed8 100644 --- a/test/cpp/features/security.feature +++ b/test/cpp/features/security.feature @@ -1,10 +1,12 @@ Feature: Security + # @sbdl security is aspect { description is "As a security engineer and security conscious developer, to have control over the security posture of my development environment, I want to have controls in place to identify and mitigate supply-chain vulnerabilities." } As a security engineer and security conscious developer, to have control over the security posture of my development environment, I want to have controls in place to identify and mitigate supply-chain vulnerabilities. Rule: Build provenance + # @sbdl build-provenance is requirement { aspect is security description is "amp-devcontainer SHALL include build provenance as specified in SLSA v1.0 Build L3." } amp-devcontainer *SHALL* include build provenance as specified in [SLSA v1.0 Build L3](https://slsa.dev/spec/v1.0/levels). Including provenance gives confidence that the container images haven't been tampered with and can be securely traced back to its source code. @@ -13,6 +15,7 @@ Feature: Security Doing so prevents several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). Rule: Signing + # @sbdl signing is requirement { aspect is security description is "amp-devcontainer SHALL cryptographically sign its released container images." } amp-devcontainer *SHALL* cryptographically sign its released container images. Cryptographically signing released container images provides integrity and authenticity guarantees. @@ -20,6 +23,7 @@ Feature: Security This helps mitigate several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). Rule: Software Bill of Materials (SBOM) + # @sbdl software-bill-of-materials-sbom is requirement { aspect is security description is "amp-devcontainer SHOULD provide a Software Bill of Materials (SBOM) for its released container images." } amp-devcontainer *SHOULD* provide a Software Bill of Materials (SBOM) for its released container images. Providing a Software Bill of Materials (SBOM) enables consumers to identify and manage security risks associated with the software components included in the container images. diff --git a/test/cpp/features/static-dynamic-analysis.feature b/test/cpp/features/static-dynamic-analysis.feature index 3254790a..a8753b0b 100644 --- a/test/cpp/features/static-dynamic-analysis.feature +++ b/test/cpp/features/static-dynamic-analysis.feature @@ -1,10 +1,12 @@ Feature: Static and dynamic analysis + # @sbdl static-and-dynamic-analysis is aspect { description is "As a software craftsperson, to maintain consistent, high-quality and bug-free code, I want my source code to be statically and dynamically analyzed." } As a software craftsperson, to maintain consistent, high-quality and bug-free code, I want my source code to be statically and dynamically analyzed. Rule: Code formatting + # @sbdl code-formatting is requirement { aspect is static-and-dynamic-analysis description is "amp-devcontainer MAY provide code formatting tools for the primary programming language(s) used within the container." } amp-devcontainer *MAY* provide code formatting tools for the primary programming language(s) used within the container. Providing code formatting tools helps maintain a consistent coding style across the codebase, improving readability and reducing friction during code reviews. @@ -13,12 +15,14 @@ Feature: Static and dynamic analysis @flavor:cpp @fixme Scenario: Format source code according to a formatting style + # @sbdl format-source-code-cpp is test { requirement is code-formatting } Given the file "clang-tools/unformatted.cpp" is opened in the editor When the active document is formatted And the active document is saved Then the contents of "clang-tools/unformatted.cpp" should match the contents of "clang-tools/formatted.cpp" Rule: Static analysis + # @sbdl static-analysis is requirement { aspect is static-and-dynamic-analysis description is "amp-devcontainer MAY provide static analysis tools for the primary programming language(s) used within the container." } amp-devcontainer *MAY* provide static analysis tools for the primary programming language(s) used within the container. Providing static analysis tools helps identify potential issues in the code before it is executed, improving code quality and reducing the likelihood of runtime errors. @@ -26,6 +30,7 @@ Feature: Static and dynamic analysis By integrating static analysis tools into the development environment, developers can catch issues before they become more significant problems, streamlining the development workflow and improving overall code quality. Rule: Coverage analysis + # @sbdl coverage-analysis is requirement { aspect is static-and-dynamic-analysis description is "amp-devcontainer SHOULD provide code coverage analysis tools for the primary programming language(s) used within the container." } amp-devcontainer *SHOULD* provide code coverage analysis tools for the primary programming language(s) used within the container. Providing code coverage analysis tools helps assess the effectiveness of the existing test suite by measuring how much of the code is exercised by the tests. @@ -33,6 +38,7 @@ Feature: Static and dynamic analysis By integrating code coverage analysis tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. Rule: Mutation testing + # @sbdl mutation-testing is requirement { aspect is static-and-dynamic-analysis description is "amp-devcontainer MAY provide mutation testing tools for the primary programming language(s) used within the container." } amp-devcontainer *MAY* provide mutation testing tools for the primary programming language(s) used within the container. Providing mutation testing tools helps assess the effectiveness of the existing test suite by introducing small changes (mutations) to the code and checking if the tests can detect these changes. @@ -40,6 +46,7 @@ Feature: Static and dynamic analysis By integrating mutation testing tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. Rule: Fuzz testing + # @sbdl fuzz-testing is requirement { aspect is static-and-dynamic-analysis description is "amp-devcontainer MAY provide fuzz testing tools for the primary programming language(s) used within the container." } amp-devcontainer *MAY* provide fuzz testing tools for the primary programming language(s) used within the container. Providing fuzz testing tools helps identify potential security vulnerabilities and robustness issues in the code by automatically generating and executing a large number of random inputs. diff --git a/test/cpp/integration-tests.bats b/test/cpp/integration-tests.bats index 7ecfcdde..98c6fd32 100644 --- a/test/cpp/integration-tests.bats +++ b/test/cpp/integration-tests.bats @@ -29,6 +29,7 @@ teardown() { # are aligned in terms of major and minor versions. # bats test_tags=compatibility,version,clang +# @sbdl cpp-version-clang is test { aspect is compatibility } @test "clang toolchain versions should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for clang) @@ -39,6 +40,7 @@ teardown() { } # bats test_tags=compatibility,version,gcc +# @sbdl cpp-version-gcc is test { aspect is compatibility } @test "host gcc toolchain versions and alternatives should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for g++) @@ -49,6 +51,7 @@ teardown() { } # bats test_tags=compatibility,version,hostgccarmgcc +# @sbdl cpp-version-host-arm-gcc is test { aspect is compatibility } @test "host and embedded gcc toolchain versions should be the same major and minor version" { EXPECTED_MAJOR_MINOR_VERSION=$(get_expected_semver_for g++ | cut -d. -f1,2) INSTALLED_MAJOR_MINOR_VERSION=$(arm-none-eabi-gcc -dumpfullversion | cut -d. -f1,2) @@ -56,6 +59,7 @@ teardown() { } # bats test_tags=compatibility,version,tools +# @sbdl cpp-version-tools is test { aspect is compatibility } @test "supporting tool versions should be aligned with expected versions" { for TOOL in gdb gdb-multiarch git ninja; do EXPECTED_VERSION=$(get_expected_semver_for ${TOOL}) @@ -73,6 +77,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-host-gcc is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "valid code input should result in working executable using host compiler" { cmake --preset gcc cmake --build --preset gcc @@ -83,6 +88,7 @@ teardown() { } # bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# @sbdl cpp-compile-arm-cortex is test { requirement is compile-for-arm-cortex-target-architecture } @test "valid code input should result in elf executable using arm-none-eabi compiler" { cmake --preset gcc-arm-none-eabi cmake --build --preset gcc-arm-none-eabi @@ -93,6 +99,7 @@ teardown() { } # bats test_tags=compilation,compile-for-microsoft-windows-operating-system +# @sbdl cpp-compile-windows is test { requirement is compile-for-microsoft-windows-operating-system } @test "valid code input should result in Windows executable using clang-cl compiler" { install_win_sdk_when_ci_unset @@ -101,6 +108,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-database is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "compilation database should be generated on CMake configure" { cmake --preset gcc assert [ -e build/gcc/compile_commands.json ] @@ -110,17 +118,20 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-host-fail is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "invalid code input should result in failing build" { cmake --preset gcc run ! cmake --build --preset gcc-fail } # bats test_tags=compilation,compilation-cache +# @sbdl cpp-ccache-gcc is test { requirement is compilation-cache } @test "using ccache as a compiler launcher should result in cached build using gcc compiler" { configure_and_build_with_ccache gcc } # bats test_tags=compilation,compilation-cache +# @sbdl cpp-ccache-clang-cl is test { requirement is compilation-cache } @test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" { install_win_sdk_when_ci_unset @@ -128,6 +139,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-clang-tidy is test { requirement is static-analysis } @test "running clang-tidy as part of the build should result in warning diagnostics" { cmake --preset clang @@ -137,6 +149,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-iwyu is test { requirement is static-analysis } @test "running include-what-you-use as part of the build should result in warning diagnostics" { cmake --preset clang @@ -146,6 +159,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,code-formatting +# @sbdl cpp-clang-format is test { requirement is code-formatting } @test "running clang-format should result in re-formatted code" { run clang-format clang-tools/unformatted.cpp assert_success @@ -153,6 +167,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,coverage-analysis +# @sbdl cpp-coverage is test { requirement is coverage-analysis } @test "coverage information should be generated when running a testsuite" { cmake --preset coverage cmake --build --preset coverage @@ -167,6 +182,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,fuzz-testing +# @sbdl cpp-fuzz-testing is test { requirement is fuzz-testing } @test "crashes should be detected when fuzzing an executable" { cmake --preset clang cmake --build --preset fuzzing @@ -177,6 +193,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,mutation-testing +# @sbdl cpp-mutation-testing is test { requirement is mutation-testing } @test "a mutation score should be calculated when mutation testing a testsuite" { cmake --preset mutation cmake --build --preset mutation @@ -186,6 +203,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-clangd-check is test { requirement is static-analysis } @test "clangd should be able to analyze source files" { run clangd --check=gcc/main.cpp assert_success @@ -193,12 +211,14 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-clangd-start is test { requirement is static-analysis } @test "clangd should start with a specified compile commands path" { run timeout 1s clangd --compile-commands-dir=/root/.amp refute_output --partial "Path specified by --compile-commands-dir does not exist. The argument will be ignored." } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-lld is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "using lld as an alternative linker should result in working host executable" { cmake --preset gcc cmake --build --preset gcc-lld @@ -212,16 +232,19 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-sanitizers-gcc is test { requirement is static-analysis } @test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" { build_and_run_with_sanitizers gcc } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl cpp-sanitizers-clang is test { requirement is static-analysis } @test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" { build_and_run_with_sanitizers clang } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-conan is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "using Conan as package manager should resolve external dependencies" { pushd package-managers/conan @@ -234,6 +257,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl cpp-compile-cpm is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "using CPM as package manager should resolve external dependencies" { cmake --preset cpm cmake --build --preset cpm diff --git a/test/rust/integration-tests.bats b/test/rust/integration-tests.bats index be5c59f8..0988d48d 100644 --- a/test/rust/integration-tests.bats +++ b/test/rust/integration-tests.bats @@ -16,6 +16,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl rust-compile-host is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "valid code input should result in working executable targeting the host architecture" { rustc --out-dir build rust/hello.rs @@ -25,6 +26,7 @@ teardown() { } # bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# @sbdl rust-compile-arm-cortex-m is test { requirement is compile-for-arm-cortex-target-architecture } @test "valid code input should result in working elf executable targeting the cortex-m architecture" { pushd cortex-m @@ -39,6 +41,7 @@ teardown() { } # bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# @sbdl rust-compile-arm-cortex-mf is test { requirement is compile-for-arm-cortex-target-architecture } @test "valid code input should result in working elf executable targeting the cortex-mf architecture" { pushd cortex-mf @@ -53,6 +56,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl rust-cargo-run is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "using cargo run should result in working executable" { pushd cargo @@ -64,6 +68,7 @@ teardown() { } # bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# @sbdl rust-compile-fail is test { requirement is compile-for-container-host-architecture-and-operating-system } @test "invalid code input should result in failing build" { run rustc rust/fail.rs assert_failure @@ -71,6 +76,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,static-analysis +# @sbdl rust-clippy is test { requirement is static-analysis } @test "running clippy should result in warning diagnostics" { pushd clippy @@ -82,6 +88,7 @@ teardown() { } # bats test_tags=static-and-dynamic-analysis,code-formatting +# @sbdl rust-rustfmt is test { requirement is code-formatting } @test "running rustfmt should result in re-formatted code" { run rustfmt --color=never --check rust/unformatted.rs assert_failure @@ -96,6 +103,7 @@ EOF } # bats test_tags=static-and-dynamic-analysis,coverage-analysis +# @sbdl rust-coverage is test { requirement is coverage-analysis } @test "coverage information should be generated when running a testsuite" { pushd test @@ -113,6 +121,7 @@ EOF } # bats test_tags=static-and-dynamic-analysis,mutation-testing +# @sbdl rust-mutation-testing is test { requirement is mutation-testing } @test "mutation testing a test executable should be supported" { pushd test