feat(cli): generate baseline of known failures

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 - Display captured test output on assertion failures when `--show-output` is enabled (#637)
+- `--generate-baseline <file>` flag (and `BASHUNIT_BASELINE_GENERATE` env var) writes an XML baseline of the run's failed/risky/incomplete tests, and `--use-baseline <file>` (`BASHUNIT_BASELINE_USE`) ignores those known issues so only newly introduced ones fail the run (#284)
 
 ### Changed
 - Speed up coverage report generation by collapsing the per-line non-executable pattern checks in `bashunit::coverage::is_executable_line` into a single combined `grep` invocation (#636)

bashunit

@@ -76,6 +76,7 @@ source "$BASHUNIT_ROOT_DIR/src/watch.sh"
 source "$BASHUNIT_ROOT_DIR/src/assertions.sh"
 source "$BASHUNIT_ROOT_DIR/src/doc.sh"
 source "$BASHUNIT_ROOT_DIR/src/reports.sh"
+source "$BASHUNIT_ROOT_DIR/src/baseline.sh"
 source "$BASHUNIT_ROOT_DIR/src/runner.sh"
 source "$BASHUNIT_ROOT_DIR/src/bashunit.sh"
 source "$BASHUNIT_ROOT_DIR/src/init.sh"

docs/command-line.md

@@ -67,6 +67,8 @@ bashunit test tests/ --parallel --simple
 | `-p, --parallel`               | Run tests in parallel                            |
 | `--no-parallel`                | Run tests sequentially                           |
 | `-r, --report-html <file>`     | Write HTML report                                |
+| `--generate-baseline <file>`   | Write a baseline XML of failed/risky/incomplete tests |
+| `--use-baseline <file>`        | Ignore tests listed in the baseline              |
 | `-R, --run-all`                | Run all assertions (don't stop on first failure) |
 | `-s, --simple`                 | Simple output (dots)                             |
 | `--detailed`                   | Detailed output (default)                        |
@@ -366,6 +368,21 @@ bashunit test tests/ --log-gha gha.log && cat gha.log
 
 The `--log-gha` flag writes GitHub Actions workflow commands (`::error`, `::warning`, `::notice`) for failed, risky and incomplete tests. When streamed to stdout on a runner, they appear as inline annotations in the "Files changed" tab of a pull request.
 
+### Baseline
+
+Capture the run's existing failed, risky and incomplete tests so they no longer fail the suite, allowing you to focus on newly introduced issues.
+
+::: code-group
+```bash [Generate]
+bashunit test tests/ --generate-baseline baseline.xml
+```
+```bash [Use]
+bashunit test tests/ --use-baseline baseline.xml
+```
+:::
+
+When `--generate-baseline` is set, the run exits with code 0 even if tests failed: the failures are recorded in the XML for later use. When `--use-baseline` is set, any failed/risky/incomplete test that matches an entry in the baseline (by file, name and status) is reported as `baselined` and excluded from the failure count. Newly failing tests still fail the run.
+
 ### Show Output on Failure
 
 > `bashunit test --show-output`

docs/configuration.md

@@ -260,6 +260,22 @@ BASHUNIT_REPORT_HTML=report.html
 ```
 :::
 
+## Baseline
+
+> `BASHUNIT_BASELINE_GENERATE=file`
+> `BASHUNIT_BASELINE_USE=file`
+
+`BASHUNIT_BASELINE_GENERATE` writes an XML baseline of every failed, risky or incomplete test in the current run. The run then exits with code 0 so the baseline can be committed.
+
+`BASHUNIT_BASELINE_USE` loads a previously generated baseline. Tests whose file, name and status all match a baseline entry are reported as `baselined` and excluded from the failure count. Tests not present in the baseline still fail the run.
+
+::: code-group
+```bash [Example]
+BASHUNIT_BASELINE_GENERATE=baseline.xml
+BASHUNIT_BASELINE_USE=baseline.xml
+```
+:::
+
 ## Bootstrap
 
 > `BASHUNIT_BOOTSTRAP=file`

src/baseline.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+_BASHUNIT_BASELINE_FILES=()
+_BASHUNIT_BASELINE_NAMES=()
+_BASHUNIT_BASELINE_STATUSES=()
+
+##
+# Escape XML special characters for use inside attribute values.
+# Arguments: $1 - text
+##
+function bashunit::baseline::__xml_escape_attr() {
+  # sed handles & and \" literals consistently across Bash versions; bash
+  # parameter expansion with patsub_replacement (5.2+) treats & in the
+  # replacement specially, which would corrupt entity strings.
+  printf '%s' "$1" \
+    | sed -e 's/&/\&/g' \
+      -e 's/</\&lt;/g' \
+      -e 's/>/\&gt;/g' \
+      -e 's/"/\&quot;/g' \
+      -e "s/'/\&apos;/g"
+}
+
+##
+# Decode the XML entities used by bashunit::baseline::__xml_escape_attr.
+# Arguments: $1 - text
+##
+function bashunit::baseline::__xml_unescape_attr() {
+  # &amp; must be decoded last so we don't double-decode.
+  printf '%s' "$1" \
+    | sed -e 's/&lt;/</g' \
+      -e 's/&gt;/>/g' \
+      -e 's/&quot;/"/g' \
+      -e "s/&apos;/'/g" \
+      -e 's/&amp;/\&/g'
+}
+
+##
+# Generate a baseline XML file listing the current run's failed/risky/incomplete tests.
+# Arguments: $1 - output file path
+##
+function bashunit::baseline::generate() {
+  local output_file="$1"
+
+  {
+    echo '<?xml version="1.0" encoding="UTF-8"?>'
+    echo '<baseline version="1.0">'
+
+    local i status file name
+    for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do
+      status="${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}"
+
+      case "$status" in
+      failed | risky | incomplete) ;;
+      *) continue ;;
+      esac
+
+      file="$(bashunit::baseline::__xml_escape_attr "${_BASHUNIT_REPORTS_TEST_FILES[$i]:-}")"
+      name="$(bashunit::baseline::__xml_escape_attr "${_BASHUNIT_REPORTS_TEST_NAMES[$i]:-}")"
+      echo "  <test file=\"$file\" name=\"$name\" status=\"$status\"/>"
+    done
+
+    echo '</baseline>'
+  } >"$output_file"
+}
+
+##
+# Load entries from a baseline XML file into the lookup arrays.
+# Arguments: $1 - input file path
+# Returns: 0 success, 1 if file missing
+##
+function bashunit::baseline::load() {
+  local input_file="$1"
+
+  if [ ! -f "$input_file" ]; then
+    return 1
+  fi
+
+  _BASHUNIT_BASELINE_FILES=()
+  _BASHUNIT_BASELINE_NAMES=()
+  _BASHUNIT_BASELINE_STATUSES=()
+
+  local line file name status
+  while IFS= read -r line; do
+    case "$line" in
+    *"<test "*)
+      file=""
+      name=""
+      status=""
+
+      case "$line" in
+      *file=\"*)
+        file="${line#*file=\"}"
+        file="${file%%\"*}"
+        ;;
+      esac
+      case "$line" in
+      *name=\"*)
+        name="${line#*name=\"}"
+        name="${name%%\"*}"
+        ;;
+      esac
+      case "$line" in
+      *status=\"*)
+        status="${line#*status=\"}"
+        status="${status%%\"*}"
+        ;;
+      esac
+
+      file="$(bashunit::baseline::__xml_unescape_attr "$file")"
+      name="$(bashunit::baseline::__xml_unescape_attr "$name")"
+
+      _BASHUNIT_BASELINE_FILES[${#_BASHUNIT_BASELINE_FILES[@]}]="$file"
+      _BASHUNIT_BASELINE_NAMES[${#_BASHUNIT_BASELINE_NAMES[@]}]="$name"
+      _BASHUNIT_BASELINE_STATUSES[${#_BASHUNIT_BASELINE_STATUSES[@]}]="$status"
+      ;;
+    esac
+  done <"$input_file"
+}
+
+##
+# Whether the run should consult the loaded baseline to suppress matched issues.
+# Returns: 0 if --use-baseline is active, 1 otherwise
+##
+function bashunit::baseline::is_use_enabled() {
+  [ -n "${BASHUNIT_BASELINE_USE:-}" ]
+}
+
+##
+# Suppress a failing/risky/incomplete test if it matches an entry in the baseline.
+# Records a "baselined" entry in the report arrays and increments the baselined counter.
+# Arguments: $1 - file, $2 - label, $3 - status, $4 - duration, $5 - assertions
+# Returns: 0 if matched and suppressed, 1 otherwise (caller should proceed normally)
+##
+function bashunit::baseline::match_and_record() {
+  local file="$1"
+  local label="$2"
+  local status="$3"
+  local duration="${4:-0}"
+  local assertions="${5:-0}"
+
+  bashunit::baseline::is_use_enabled || return 1
+  bashunit::baseline::contains "$file" "$label" "$status" || return 1
+
+  bashunit::state::add_tests_baselined
+  # Force-track the baselined entry regardless of report flags.
+  local _prev="${BASHUNIT_BASELINE_GENERATE:-}"
+  BASHUNIT_BASELINE_GENERATE="${BASHUNIT_BASELINE_GENERATE:-baselined}"
+  bashunit::reports::add_test "$file" "$label" "$duration" "$assertions" "baselined"
+  BASHUNIT_BASELINE_GENERATE="$_prev"
+  return 0
+}
+
+##
+# Check whether a (file, name, status) triple exists in the loaded baseline.
+# Arguments: $1 - file, $2 - test name, $3 - status
+# Returns: 0 if present, 1 otherwise
+##
+function bashunit::baseline::contains() {
+  local file="$1"
+  local name="$2"
+  local status="$3"
+
+  local i
+  for i in "${!_BASHUNIT_BASELINE_FILES[@]}"; do
+    if [ "${_BASHUNIT_BASELINE_FILES[$i]}" = "$file" ] &&
+      [ "${_BASHUNIT_BASELINE_NAMES[$i]}" = "$name" ] &&
+      [ "${_BASHUNIT_BASELINE_STATUSES[$i]}" = "$status" ]; then
+      return 0
+    fi
+  done
+
+  return 1
+}

src/console_header.sh

@@ -113,6 +113,8 @@ Options:
   -p, --parallel              Run tests in parallel (unlimited concurrency)
   --no-parallel               Run tests sequentially
   -r, --report-html <file>    Write HTML report
+  --generate-baseline <file>  Write a baseline XML of current failed/risky/incomplete tests
+  --use-baseline <file>       Ignore tests listed in the baseline (don't count as failures)
   -s, --simple                Simple output (dots)
   --detailed                  Detailed output (default)
   --output <format>           Output format: tap (TAP version 13)

src/console_results.sh

@@ -31,6 +31,7 @@ function bashunit::console_results::render_result() {
   local tests_snapshot=$_BASHUNIT_TESTS_SNAPSHOT
   local tests_failed=$_BASHUNIT_TESTS_FAILED
   local tests_risky=$_BASHUNIT_TESTS_RISKY
+  local tests_baselined=$_BASHUNIT_TESTS_BASELINED
   local assertions_passed=$_BASHUNIT_ASSERTIONS_PASSED
   local assertions_skipped=$_BASHUNIT_ASSERTIONS_SKIPPED
   local assertions_incomplete=$_BASHUNIT_ASSERTIONS_INCOMPLETE
@@ -44,6 +45,7 @@ function bashunit::console_results::render_result() {
   total_tests=$((total_tests + tests_snapshot))
   total_tests=$((total_tests + tests_failed))
   total_tests=$((total_tests + tests_risky))
+  total_tests=$((total_tests + tests_baselined))
 
   local total_assertions=0
   total_assertions=$((total_assertions + assertions_passed))
@@ -71,6 +73,9 @@ function bashunit::console_results::render_result() {
   if [ "$tests_risky" -gt 0 ]; then
     printf " %s%s risky%s," "$_BASHUNIT_COLOR_RISKY" "$tests_risky" "$_BASHUNIT_COLOR_DEFAULT"
   fi
+  if [ "$tests_baselined" -gt 0 ]; then
+    printf " %s%s baselined%s," "$_BASHUNIT_COLOR_SKIPPED" "$tests_baselined" "$_BASHUNIT_COLOR_DEFAULT"
+  fi
   printf " %s total\n" "$total_tests"
 
   printf "%sAssertions:%s" "$_BASHUNIT_COLOR_FAINT" "$_BASHUNIT_COLOR_DEFAULT"

src/env.sh

@@ -16,6 +16,8 @@ _BASHUNIT_DEFAULT_DEV_LOG=""
 _BASHUNIT_DEFAULT_LOG_JUNIT=""
 _BASHUNIT_DEFAULT_LOG_GHA=""
 _BASHUNIT_DEFAULT_REPORT_HTML=""
+_BASHUNIT_DEFAULT_BASELINE_GENERATE=""
+_BASHUNIT_DEFAULT_BASELINE_USE=""
 
 # Coverage defaults (following kcov, bashcov, SimpleCov conventions)
 _BASHUNIT_DEFAULT_COVERAGE="false"
@@ -34,6 +36,8 @@ _BASHUNIT_DEFAULT_COVERAGE_THRESHOLD_HIGH="80"
 : "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_BASHUNIT_DEFAULT_LOG_JUNIT}}"
 : "${BASHUNIT_LOG_GHA:=${LOG_GHA:=$_BASHUNIT_DEFAULT_LOG_GHA}}"
 : "${BASHUNIT_REPORT_HTML:=${REPORT_HTML:=$_BASHUNIT_DEFAULT_REPORT_HTML}}"
+: "${BASHUNIT_BASELINE_GENERATE:=${BASELINE_GENERATE:=$_BASHUNIT_DEFAULT_BASELINE_GENERATE}}"
+: "${BASHUNIT_BASELINE_USE:=${BASELINE_USE:=$_BASHUNIT_DEFAULT_BASELINE_USE}}"
 
 # Coverage
 : "${BASHUNIT_COVERAGE:=${COVERAGE:=$_BASHUNIT_DEFAULT_COVERAGE}}"
@@ -240,6 +244,8 @@ function bashunit::env::print_verbose() {
     "BASHUNIT_LOG_JUNIT"
     "BASHUNIT_LOG_GHA"
     "BASHUNIT_REPORT_HTML"
+    "BASHUNIT_BASELINE_GENERATE"
+    "BASHUNIT_BASELINE_USE"
     "BASHUNIT_PARALLEL_RUN"
     "BASHUNIT_SHOW_HEADER"
     "BASHUNIT_HEADER_ASCII_ART"

src/main.sh

@@ -104,6 +104,14 @@ function bashunit::main::cmd_test() {
       export BASHUNIT_REPORT_HTML="$2"
       shift
       ;;
+    --generate-baseline)
+      export BASHUNIT_BASELINE_GENERATE="$2"
+      shift
+      ;;
+    --use-baseline)
+      export BASHUNIT_BASELINE_USE="$2"
+      shift
+      ;;
     --no-output)
       export BASHUNIT_NO_OUTPUT=true
       ;;
@@ -673,6 +681,14 @@ function bashunit::main::exec_tests() {
     printf '%*s\n' "$TERMINAL_WIDTH" '' | tr ' ' '#'
   fi
 
+  if [ -n "${BASHUNIT_BASELINE_USE:-}" ]; then
+    if ! bashunit::baseline::load "$BASHUNIT_BASELINE_USE"; then
+      printf "%sError: baseline file not found: %s%s\n" \
+        "${_BASHUNIT_COLOR_FAILED}" "$BASHUNIT_BASELINE_USE" "${_BASHUNIT_COLOR_DEFAULT}"
+      exit 1
+    fi
+  fi
+
   bashunit::runner::load_test_files "$filter" "$tag_filter" "$exclude_tag_filter" "${test_files[@]}"
 
   if bashunit::parallel::is_enabled; then
@@ -704,6 +720,12 @@ function bashunit::main::exec_tests() {
     bashunit::reports::generate_report_html "$BASHUNIT_REPORT_HTML"
   fi
 
+  if [ -n "${BASHUNIT_BASELINE_GENERATE:-}" ]; then
+    bashunit::baseline::generate "$BASHUNIT_BASELINE_GENERATE"
+    # Listed issues are intentional baseline; treat run as success.
+    exit_code=0
+  fi
+
   # Generate coverage report if enabled
   if bashunit::env::is_coverage_enabled; then
     # Aggregate per-process coverage data from parallel runs

src/reports.sh

@@ -37,7 +37,8 @@ function bashunit::reports::add_test() {
   {
     [ -n "${BASHUNIT_LOG_JUNIT:-}" ] ||
       [ -n "${BASHUNIT_REPORT_HTML:-}" ] ||
-      [ -n "${BASHUNIT_LOG_GHA:-}" ]
+      [ -n "${BASHUNIT_LOG_GHA:-}" ] ||
+      [ -n "${BASHUNIT_BASELINE_GENERATE:-}" ]
   } || return 0
 
   local file="$1"