diff --git a/Sources/Testing/Traits/CoverageTrait.swift b/Sources/Testing/Traits/CoverageTrait.swift new file mode 100644 index 000000000..dd9867c78 --- /dev/null +++ b/Sources/Testing/Traits/CoverageTrait.swift @@ -0,0 +1,241 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +internal import _TestingInternals + +// MARK: - Coverage Detection + +/// Check if coverage instrumentation is enabled for this test run. +/// +/// This is determined by checking for the presence of coverage-related +/// environment variables. +private let _coverageEnabled: Bool = { + // Check for LLVM_PROFILE_FILE environment variable (set by swift test) + Environment.variable(named: "LLVM_PROFILE_FILE") != nil +}() + +/// Get the output directory for coverage files. +private let _coverageOutputDirectory: String = { + if let dir = Environment.variable(named: "COVERAGE_OUTPUT_DIR") { + return dir + } + if let cwd = swt_getEarlyCWD() { + return String(cString: cwd) + } + return "." +}() + +// MARK: - CoverageTrait + +/// A trait that collects per-test code coverage. +/// +/// When this trait is applied to a test or suite, each test case gets its own +/// coverage profile file, enabling analysis of which code paths each test +/// exercises. +/// +/// - Important: This trait automatically enforces serialized execution because +/// LLVM profile counters are shared global state. Tests with this trait will +/// run serially regardless of other parallelization settings. +/// +/// ## Usage +/// +/// Apply to a single test: +/// +/// ```swift +/// @Test(.coverage) +/// func testFeature() { +/// // Coverage is written to coverage-testFeature.profraw +/// } +/// ``` +/// +/// Apply to an entire suite: +/// +/// ```swift +/// @Suite(.coverage) +/// struct MyTests { +/// @Test func test1() { ... } +/// @Test func test2() { ... } +/// } +/// ``` +/// +/// ## Requirements +/// +/// Per-test coverage requires building and running tests with coverage enabled: +/// +/// ```bash +/// swift test --enable-code-coverage +/// ``` +/// +/// ## Output +/// +/// Coverage files are written to `$COVERAGE_OUTPUT_DIR` (or current directory): +/// +/// - `coverage-testName.profraw` for each test +/// +/// Merge and view results: +/// +/// ```bash +/// xcrun llvm-profdata merge -sparse coverage-*.profraw -o merged.profdata +/// xcrun llvm-cov report .build/debug/TestBundle -instr-profile=merged.profdata +/// ``` +@_spi(Experimental) +public struct CoverageTrait: TestTrait, SuiteTrait { + /// Whether to reset coverage counters before each test. + /// + /// When `true` (default), each test's coverage file only contains the + /// coverage from that specific test. When `false`, coverage accumulates + /// across tests. + public var isolatesTests: Bool + + /// The directory where coverage files are written. + /// + /// Defaults to `COVERAGE_OUTPUT_DIR` environment variable or current + /// working directory. + public var outputDirectory: String + + /// Create a coverage trait. + /// + /// - Parameters: + /// - isolatesTests: Whether to reset counters before each test. + /// - outputDirectory: Directory for coverage files. + public init( + isolatesTests: Bool = true, + outputDirectory: String? = nil + ) { + self.isolatesTests = isolatesTests + self.outputDirectory = outputDirectory ?? _coverageOutputDirectory + } + + public var isRecursive: Bool { true } +} + +// MARK: - TestScoping + +extension CoverageTrait: TestScoping { + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + // When applied to a test function, provide scope to the test function + // itself (not individual test cases) so we can disable parallelization + // at the suite/function level. + test.isSuite || testCase == nil ? self : nil + } + + public func provideScope( + for test: Test, + testCase: Test.Case?, + performing function: @Sendable () async throws -> Void + ) async throws { + // Skip coverage collection if not enabled + guard _coverageEnabled else { + try await function() + return + } + + // Enforce serialization - coverage requires serial execution because + // LLVM profile counters are shared global state across all threads. + guard var configuration = Configuration.current else { + try await function() + return + } + configuration.isParallelizationEnabled = false + + try await Configuration.withCurrent(configuration) { + // Reset counters to isolate this test's coverage + if isolatesTests { + __llvm_profile_reset_counters() + } + + // Run the test + do { + try await function() + } catch { + // Write coverage even if test fails + writeCoverage(for: test, testCase: testCase) + throw error + } + + // Write coverage for this test + writeCoverage(for: test, testCase: testCase) + } + } + + private func writeCoverage(for test: Test, testCase: Test.Case?) { + let filename = coverageFilename(for: test, testCase: testCase) + + filename.withCString { cString in + __llvm_profile_set_filename(cString) + } + + _ = __llvm_profile_write_file() + } + + private func coverageFilename(for test: Test, testCase: Test.Case?) -> String { + var name = test.name + if let testCase { + // Include test case ID for parameterized tests + name += "-\(testCase.id)" + } + + // Sanitize for filesystem - replace problematic characters + var sanitized = "" + for char in name { + switch char { + case "/", ":", " ", "(", ")", ",", "\\", "<", ">", "\"", "|", "?", "*": + sanitized.append("_") + default: + sanitized.append(char) + } + } + + return "\(outputDirectory)/coverage-\(sanitized).profraw" + } +} + +// MARK: - Trait Extension + +@_spi(Experimental) +extension Trait where Self == CoverageTrait { + /// A trait that collects per-test code coverage. + /// + /// Apply this trait to tests or suites to generate individual coverage + /// profiles for each test case. + /// + /// ```swift + /// @Test(.coverage) + /// func testFeature() { ... } + /// ``` + public static var coverage: Self { + Self() + } + + /// A trait that collects per-test code coverage with custom options. + /// + /// - Parameters: + /// - isolatesTests: Whether to reset counters between tests. + /// - outputDirectory: Directory for coverage files. + public static func coverage( + isolatesTests: Bool = true, + outputDirectory: String? = nil + ) -> Self { + Self(isolatesTests: isolatesTests, outputDirectory: outputDirectory) + } +} + +// MARK: - Coverage Utilities + +@_spi(Experimental) +extension CoverageTrait { + /// Whether coverage instrumentation is available for this test run. + /// + /// Returns `true` if the test binary was compiled with coverage + /// instrumentation (`--enable-code-coverage`). + public static var isAvailable: Bool { + _coverageEnabled + } +} diff --git a/Sources/_TestingInternals/include/InstrProfiling.h b/Sources/_TestingInternals/include/InstrProfiling.h new file mode 100644 index 000000000..4fe5a9b5b --- /dev/null +++ b/Sources/_TestingInternals/include/InstrProfiling.h @@ -0,0 +1,88 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !defined(SWT_INSTR_PROFILING_H) +#define SWT_INSTR_PROFILING_H + +#include "Defines.h" +#include "Includes.h" + +SWT_ASSUME_NONNULL_BEGIN + +/// @defgroup LLVM Profile Runtime Interface +/// @{ +/// +/// These functions are provided by the LLVM profile runtime when code is +/// compiled with coverage instrumentation (-profile-generate or +/// --enable-code-coverage in Swift). +/// +/// Reference: https://github.com/llvm/llvm-project/blob/main/compiler-rt/include/profile/instr_prof_interface.h + +/// Reset all profile counters to zero. +/// +/// Call this before running a section of code to isolate its coverage from +/// previously executed code. +/// +/// @note This function is only available when coverage instrumentation is +/// enabled. Use swt_profilerRuntimeAvailable() to check availability. +SWT_EXTERN void __llvm_profile_reset_counters(void); + +/// Write the current profile data to the configured file. +/// +/// @returns 0 on success, non-zero on failure. +/// +/// @note The output filename is determined by the LLVM_PROFILE_FILE environment +/// variable or can be set with __llvm_profile_set_filename(). +SWT_EXTERN int __llvm_profile_write_file(void); + +/// Set the filename for subsequent profile writes. +/// +/// @param filename The path to write profile data to. This string must remain +/// valid until the next call to this function or until the profile is written. +/// Pass NULL to restore the default filename behavior. +SWT_EXTERN void __llvm_profile_set_filename(const char *_Nullable filename); + +/// Write the current profile data and mark it as dumped. +/// +/// This function is similar to __llvm_profile_write_file(), but it also marks +/// the profile as "dumped" which prevents the automatic dump that normally +/// occurs at program exit. +/// +/// @returns 0 on success, non-zero on failure. +SWT_EXTERN int __llvm_profile_dump(void); + +/// @} + +/// Check if the LLVM profile runtime is available. +/// +/// This function uses dlsym to check for the presence of profile runtime +/// symbols. Returns true if coverage instrumentation is available. +/// +/// @note This is a runtime check because the profile symbols are only present +/// when the code was compiled with coverage instrumentation. +static inline bool swt_profilerRuntimeAvailable(void) { +#if __has_include() + // Use RTLD_DEFAULT to search all loaded images +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) + void *handle = (void *)(intptr_t)-2; // RTLD_DEFAULT on Apple/BSD +#elif defined(__linux__) || defined(__ANDROID__) + void *handle = NULL; // RTLD_DEFAULT on Linux +#else + void *handle = NULL; +#endif + return dlsym(handle, "__llvm_profile_reset_counters") != NULL; +#else + return false; +#endif +} + +SWT_ASSUME_NONNULL_END + +#endif diff --git a/Tests/TestingTests/CoverageTests.swift b/Tests/TestingTests/CoverageTests.swift new file mode 100644 index 000000000..782c9c57d --- /dev/null +++ b/Tests/TestingTests/CoverageTests.swift @@ -0,0 +1,71 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) import Testing +@testable import _TestingInternals + +@Suite("Coverage API Tests") +struct CoverageTests { + @Test("Profile functions are callable") + func profileFunctionsCallable() { + // Reset counters - should not crash + __llvm_profile_reset_counters() + print("Reset counters: OK") + + // Set a temporary filename + let filename = "/tmp/test-coverage-\(getpid()).profraw" + filename.withCString { cString in + __llvm_profile_set_filename(cString) + } + print("Set filename: \(filename)") + + // Write profile + let result = __llvm_profile_write_file() + print("Write result: \(result)") + #expect(result == 0, "Profile write should succeed") + } + + @Test("CoverageTrait availability detection") + func coverageTraitAvailability() { + print("CoverageTrait.isAvailable: \(CoverageTrait.isAvailable)") + // Just verify it doesn't crash - actual value depends on how tests are run + } +} + +// MARK: - Example usage with CoverageTrait + +/// Example code to demonstrate coverage measurement +private func exampleFunction(_ value: Int) -> String { + if value > 0 { + return "positive" + } else if value < 0 { + return "negative" + } else { + return "zero" + } +} + +@Suite("Coverage Trait Demo", .coverage(outputDirectory: "/tmp")) +struct CoverageTraitDemo { + @Test("Positive path") + func positiveValue() { + #expect(exampleFunction(5) == "positive") + } + + @Test("Negative path") + func negativeValue() { + #expect(exampleFunction(-3) == "negative") + } + + @Test("Zero path") + func zeroValue() { + #expect(exampleFunction(0) == "zero") + } +}