diff --git a/README.md b/README.md index 15bcb5e..977cbf3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,13 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based detectors currently report no confidence score. +The initial transformation strategy is `redact`, which replaces each selected +finding with an unnumbered `[ENTITY_TYPE]` placeholder. `transform` requires +explicit findings; `scan_and_transform` (or `scanAndTransform` in JavaScript) +is the explicit scan-then-transform convenience. Results include the transformed +text and an ordered record for every applied replacement, including its output +byte and code-point ranges. + ## Packages | Runtime | Distribution | Import | Status | @@ -31,12 +38,18 @@ cargo add datafog-core ``` ```rust -use datafog_core::scan; +use datafog_core::{scan, scan_and_transform, TransformationStrategy}; let findings = scan("Email jane@example.com"); assert_eq!(findings[0].entity_type, "EMAIL"); assert_eq!(findings[0].matched_text, "jane@example.com"); assert_eq!(findings[0].byte_range.start, 6); + +let result = scan_and_transform( + "Email jane@example.com", + TransformationStrategy::Redact, +).unwrap(); +assert_eq!(result.text, "Email [EMAIL]"); ``` ### Python @@ -46,12 +59,15 @@ python -m pip install datafog-core ``` ```python -from datafog_core import scan +from datafog_core import scan, scan_and_transform findings = scan("Email jane@example.com") print(findings[0].entity_type) # EMAIL print(findings[0].matched_text) # jane@example.com print(findings[0].byte_range.start) # 6 + +result = scan_and_transform("Email jane@example.com", "redact") +assert result.text == "Email [EMAIL]" ``` ### Node.js @@ -59,9 +75,10 @@ print(findings[0].byte_range.start) # 6 `@datafog/node` will install as a native package once its npm release is published. ```js -import { scan } from "@datafog/node"; +import { scan, scanAndTransform } from "@datafog/node"; console.log(scan("Email jane@example.com")); +console.log(scanAndTransform("Email jane@example.com", "redact").text); ``` The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64. @@ -71,10 +88,11 @@ The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linu `@datafog/wasm` will install from npm once its first release is published. ```js -import { init, scan } from "@datafog/wasm"; +import { init, scan, scanAndTransform } from "@datafog/wasm"; await init(); console.log(scan("Email jane@example.com")); +console.log(scanAndTransform("Email jane@example.com", "redact").text); ``` ## Development diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index 90bc453..5f1d68a 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -1,2 +1,4 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; + +export type TransformationStrategy = "redact"; diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 325979f..da8ba53 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -1,5 +1,7 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; + +export type TransformationStrategy = "redact"; export interface Finding { readonly entityType: EntityType readonly matchedText: string @@ -13,7 +15,26 @@ export interface Finding { /** Scan text for supported PII findings. */ export declare function scan(text: string): Array +/** Scan text and transform the detected findings. */ +export declare function scanAndTransform(text: string, strategy: TransformationStrategy): TransformResult + export interface TextRange { readonly start: number readonly end: number } + +/** Transform explicit findings without scanning implicitly. */ +export declare function transform(text: string, findings: Array, strategy: TransformationStrategy): TransformResult + +export interface Transformation { + readonly finding: Finding + readonly strategy: TransformationStrategy + readonly replacement: string + readonly outputByteRange: TextRange + readonly outputCodepointRange: TextRange +} + +export interface TransformResult { + readonly text: string + readonly transformations: Array +} diff --git a/bindings/node/index.js b/bindings/node/index.js index 2af7003..40a3838 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,4 +1,8 @@ -import { scan as nativeScan } from "./native.js"; +import { + scan as nativeScan, + scanAndTransform as nativeScanAndTransform, + transform as nativeTransform, +} from "./native.js"; export function scan(text) { if (typeof text !== "string") { @@ -7,3 +11,28 @@ export function scan(text) { return nativeScan(text); } + +export function transform(text, findings, strategy) { + if (typeof text !== "string") { + throw new TypeError("transform text must be a string"); + } + if (!Array.isArray(findings)) { + throw new TypeError("transform findings must be an array"); + } + if (typeof strategy !== "string") { + throw new TypeError("transform strategy must be a string"); + } + + return nativeTransform(text, findings, strategy); +} + +export function scanAndTransform(text, strategy) { + if (typeof text !== "string") { + throw new TypeError("scanAndTransform text must be a string"); + } + if (typeof strategy !== "string") { + throw new TypeError("scanAndTransform strategy must be a string"); + } + + return nativeScanAndTransform(text, strategy); +} diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 3b4733f..988782b 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -3,7 +3,7 @@ use napi::{Error, Status}; use napi_derive::napi; -#[napi(object, object_from_js = false)] +#[napi(object)] pub struct TextRange { #[napi(readonly)] pub start: u32, @@ -12,7 +12,7 @@ pub struct TextRange { pub end: u32, } -#[napi(object, object_from_js = false)] +#[napi(object)] pub struct Finding { #[napi(readonly, ts_type = "EntityType")] pub entity_type: String, @@ -36,6 +36,33 @@ pub struct Finding { pub detector_version: Option, } +#[napi(object, object_from_js = false)] +pub struct Transformation { + #[napi(readonly)] + pub finding: Finding, + + #[napi(readonly, ts_type = "TransformationStrategy")] + pub strategy: String, + + #[napi(readonly)] + pub replacement: String, + + #[napi(readonly)] + pub output_byte_range: TextRange, + + #[napi(readonly)] + pub output_codepoint_range: TextRange, +} + +#[napi(object, object_from_js = false)] +pub struct TransformResult { + #[napi(readonly)] + pub text: String, + + #[napi(readonly)] + pub transformations: Vec, +} + fn js_offset(offset: usize) -> napi::Result { u32::try_from(offset).map_err(|_| { Error::new( @@ -52,21 +79,93 @@ fn js_range(range: datafog_core::TextRange) -> napi::Result { }) } +fn js_finding(finding: datafog_core::Finding) -> napi::Result { + Ok(Finding { + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: js_range(finding.byte_range)?, + codepoint_range: js_range(finding.codepoint_range)?, + confidence: finding.confidence.map(f64::from), + detector_name: finding.detector_name, + detector_version: finding.detector_version, + }) +} + +fn core_finding(finding: Finding) -> datafog_core::Finding { + datafog_core::Finding { + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: datafog_core::TextRange { + start: finding.byte_range.start as usize, + end: finding.byte_range.end as usize, + }, + codepoint_range: datafog_core::TextRange { + start: finding.codepoint_range.start as usize, + end: finding.codepoint_range.end as usize, + }, + confidence: finding.confidence.map(|confidence| confidence as f32), + detector_name: finding.detector_name, + detector_version: finding.detector_version, + } +} + +fn core_strategy(strategy: &str) -> napi::Result { + match strategy { + "redact" => Ok(datafog_core::TransformationStrategy::Redact), + _ => Err(Error::new(Status::InvalidArg, "strategy must be 'redact'")), + } +} + +fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result { + Ok(TransformResult { + text: result.text, + transformations: result + .transformations + .into_iter() + .map(|transformation| { + Ok(Transformation { + finding: js_finding(transformation.finding)?, + strategy: match transformation.strategy { + datafog_core::TransformationStrategy::Redact => "redact".to_owned(), + }, + replacement: transformation.replacement, + output_byte_range: js_range(transformation.output_byte_range)?, + output_codepoint_range: js_range(transformation.output_codepoint_range)?, + }) + }) + .collect::>>()?, + }) +} + /// Scan text for supported PII findings. #[napi(strict, catch_unwind)] pub fn scan(text: String) -> napi::Result> { datafog_core::scan(&text) .into_iter() - .map(|finding| { - Ok(Finding { - entity_type: finding.entity_type, - matched_text: finding.matched_text, - byte_range: js_range(finding.byte_range)?, - codepoint_range: js_range(finding.codepoint_range)?, - confidence: finding.confidence.map(f64::from), - detector_name: finding.detector_name, - detector_version: finding.detector_version, - }) - }) + .map(js_finding) .collect() } + +/// Transform explicit findings without scanning implicitly. +#[napi(strict, catch_unwind)] +pub fn transform( + text: String, + findings: Vec, + #[napi(ts_arg_type = "TransformationStrategy")] strategy: String, +) -> napi::Result { + let findings = findings.into_iter().map(core_finding).collect::>(); + datafog_core::transform(&text, &findings, core_strategy(&strategy)?) + .map_err(|error| Error::new(Status::InvalidArg, error.to_string())) + .and_then(js_transform_result) +} + +/// Scan text and transform the detected findings. +#[napi(strict, catch_unwind)] +pub fn scan_and_transform( + text: String, + #[napi(ts_arg_type = "TransformationStrategy")] strategy: String, +) -> napi::Result { + datafog_core::scan_and_transform(&text, core_strategy(&strategy)?) + .map_err(|error| Error::new(Status::GenericFailure, error.to_string())) + .and_then(js_transform_result) +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 21d360a..979aa9f 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,5 +1,5 @@ use ::datafog_core as core; -use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; /// A zero-based, end-exclusive text range. @@ -24,6 +24,11 @@ impl From for TextRange { #[pymethods] impl TextRange { + #[new] + fn new(start: usize, end: usize) -> Self { + Self { start, end } + } + fn __repr__(&self) -> String { format!("TextRange(start={}, end={})", self.start, self.end) } @@ -35,7 +40,7 @@ impl TextRange { /// A piece of potentially sensitive content detected in input text. #[pyclass(frozen, skip_from_py_object)] -#[derive(Clone)] +#[derive(Clone, PartialEq)] struct Finding { #[pyo3(get)] entity_type: String, @@ -75,6 +80,36 @@ impl From for Finding { #[pymethods] impl Finding { + #[new] + #[pyo3(signature = ( + entity_type, + matched_text, + byte_range, + codepoint_range, + detector_name, + confidence=None, + detector_version=None + ))] + fn new( + entity_type: String, + matched_text: String, + byte_range: PyRef<'_, TextRange>, + codepoint_range: PyRef<'_, TextRange>, + detector_name: String, + confidence: Option, + detector_version: Option, + ) -> Self { + Self { + entity_type, + matched_text, + byte_range: byte_range.clone(), + codepoint_range: codepoint_range.clone(), + confidence, + detector_name, + detector_version, + } + } + fn __repr__(&self) -> String { format!( "Finding(entity_type={:?}, matched_text={:?}, byte_range={:?}, \ @@ -101,6 +136,115 @@ impl Finding { } } +impl Finding { + fn to_core(&self) -> core::Finding { + core::Finding { + entity_type: self.entity_type.clone(), + matched_text: self.matched_text.clone(), + byte_range: core::TextRange { + start: self.byte_range.start, + end: self.byte_range.end, + }, + codepoint_range: core::TextRange { + start: self.codepoint_range.start, + end: self.codepoint_range.end, + }, + confidence: self.confidence, + detector_name: self.detector_name.clone(), + detector_version: self.detector_version.clone(), + } + } +} + +/// One privacy transformation applied to source text. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone, PartialEq)] +struct Transformation { + #[pyo3(get)] + finding: Finding, + + #[pyo3(get)] + strategy: String, + + #[pyo3(get)] + replacement: String, + + #[pyo3(get)] + output_byte_range: TextRange, + + #[pyo3(get)] + output_codepoint_range: TextRange, +} + +impl From for Transformation { + fn from(transformation: core::Transformation) -> Self { + Self { + finding: transformation.finding.into(), + strategy: match transformation.strategy { + core::TransformationStrategy::Redact => "redact".to_owned(), + }, + replacement: transformation.replacement, + output_byte_range: transformation.output_byte_range.into(), + output_codepoint_range: transformation.output_codepoint_range.into(), + } + } +} + +#[pymethods] +impl Transformation { + fn __eq__(&self, other: PyRef<'_, Transformation>) -> bool { + self.finding.entity_type == other.finding.entity_type + && self.finding.matched_text == other.finding.matched_text + && self.finding.byte_range == other.finding.byte_range + && self.finding.codepoint_range == other.finding.codepoint_range + && self.finding.confidence == other.finding.confidence + && self.finding.detector_name == other.finding.detector_name + && self.finding.detector_version == other.finding.detector_version + && self.strategy == other.strategy + && self.replacement == other.replacement + && self.output_byte_range == other.output_byte_range + && self.output_codepoint_range == other.output_codepoint_range + } +} + +/// Text and audit records produced by a transformation. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct TransformResult { + #[pyo3(get)] + text: String, + + #[pyo3(get)] + transformations: Vec, +} + +impl From for TransformResult { + fn from(result: core::TransformResult) -> Self { + Self { + text: result.text, + transformations: result + .transformations + .into_iter() + .map(Transformation::from) + .collect(), + } + } +} + +#[pymethods] +impl TransformResult { + fn __eq__(&self, other: PyRef<'_, TransformResult>) -> bool { + self.text == other.text && self.transformations == other.transformations + } +} + +fn parse_strategy(strategy: &str) -> PyResult { + match strategy { + "redact" => Ok(core::TransformationStrategy::Redact), + _ => Err(PyValueError::new_err("strategy must be 'redact'")), + } +} + /// Scan text for supported PII findings. #[pyfunction] fn scan(text: &str) -> PyResult> { @@ -108,10 +252,40 @@ fn scan(text: &str) -> PyResult> { .map_err(|_| PyRuntimeError::new_err("unexpected Rust scan failure")) } +/// Transform explicit findings without scanning implicitly. +#[pyfunction] +fn transform( + py: Python<'_>, + text: &str, + findings: Vec>, + strategy: &str, +) -> PyResult { + let strategy = parse_strategy(strategy)?; + let core_findings: Vec = findings + .iter() + .map(|finding| finding.bind(py).borrow().to_core()) + .collect(); + core::transform(text, &core_findings, strategy) + .map(TransformResult::from) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Scan text and transform the detected findings. +#[pyfunction] +fn scan_and_transform(text: &str, strategy: &str) -> PyResult { + core::scan_and_transform(text, parse_strategy(strategy)?) + .map(TransformResult::from) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) +} + #[pymodule] fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(scan, module)?)?; + module.add_function(wrap_pyfunction!(transform, module)?)?; + module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?; Ok(()) } diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index 5178468..a62c6fd 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -5,7 +5,7 @@ import json from pathlib import Path -from datafog_core import scan +from datafog_core import Finding, TextRange, scan, scan_and_transform, transform ROOT = Path(__file__).resolve().parents[3] @@ -59,7 +59,34 @@ def main() -> None: emoji_finding = scan("👋 jane@example.com")[0] assert (emoji_finding.byte_range.start, emoji_finding.byte_range.end) == (5, 21) assert (emoji_finding.codepoint_range.start, emoji_finding.codepoint_range.end) == (2, 18) - print("Installed datafog_core wheel matches fixtures and the Finding contract.") + + text = "👋 jane@example.com and jane@example.com" + explicit = transform(text, scan(text), "redact") + convenience = scan_and_transform(text, "redact") + assert explicit == convenience + assert explicit.text == "👋 [EMAIL] and [EMAIL]" + assert len(explicit.transformations) == 2 + first = explicit.transformations[0] + assert first.replacement == "[EMAIL]" + assert (first.output_byte_range.start, first.output_byte_range.end) == (5, 12) + assert (first.output_codepoint_range.start, first.output_codepoint_range.end) == (2, 9) + + invalid = Finding( + "EMAIL", + "jane@example.com", + TextRange(5, 21), + TextRange(2, 18), + "test-detector", + confidence=2.0, + ) + try: + transform(text, [invalid], "redact") + except ValueError as error: + assert "InvalidConfidence" in str(error) + else: + raise AssertionError("invalid caller-supplied finding was accepted") + + print("Installed datafog_core wheel matches fixtures and transform contracts.") if __name__ == "__main__": diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index dfe6171..dccad4a 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -1,5 +1,6 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; +export type TransformationStrategy = "redact"; export interface TextRange { readonly start: number; @@ -16,5 +17,27 @@ export interface Finding { readonly detectorVersion?: string; } +export interface Transformation { + readonly finding: Finding; + readonly strategy: TransformationStrategy; + readonly replacement: string; + readonly outputByteRange: TextRange; + readonly outputCodepointRange: TextRange; +} + +export interface TransformResult { + readonly text: string; + readonly transformations: Transformation[]; +} + export function init(): Promise; export function scan(text: string): Finding[]; +export function transform( + text: string, + findings: Finding[], + strategy: TransformationStrategy, +): TransformResult; +export function scanAndTransform( + text: string, + strategy: TransformationStrategy, +): TransformResult; diff --git a/bindings/wasm/index.js b/bindings/wasm/index.js index 2ba8e3e..3b3e72c 100644 --- a/bindings/wasm/index.js +++ b/bindings/wasm/index.js @@ -1,4 +1,8 @@ -import initWasm, { scan as scanWasm } from "./dist/datafog_wasm.js"; +import initWasm, { + scan as scanWasm, + scan_and_transform as scanAndTransformWasm, + transform as transformWasm, +} from "./dist/datafog_wasm.js"; let initialization; let initialized = false; @@ -31,3 +35,46 @@ export function scan(text) { return scanWasm(text); } + +function assertInitialized(operation) { + if (!initialized) { + throw new Error(`Call and await init() before ${operation}().`); + } +} + +function assertStrategy(strategy) { + if (strategy !== "redact") { + throw new TypeError("strategy must be 'redact'"); + } +} + +export function transform(text, findings, strategy) { + if (typeof text !== "string") { + throw new TypeError("transform text must be a string"); + } + if (!Array.isArray(findings)) { + throw new TypeError("transform findings must be an array"); + } + assertStrategy(strategy); + assertInitialized("transform"); + + try { + return transformWasm(text, findings, strategy); + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)); + } +} + +export function scanAndTransform(text, strategy) { + if (typeof text !== "string") { + throw new TypeError("scanAndTransform text must be a string"); + } + assertStrategy(strategy); + assertInitialized("scanAndTransform"); + + try { + return scanAndTransformWasm(text, strategy); + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)); + } +} diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 304184e..f6c8f70 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -1,7 +1,7 @@ -use serde::Serialize; +use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] struct TextRange { start: usize, end: usize, @@ -16,7 +16,7 @@ impl From for TextRange { } } -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase")] struct Finding { entity_type: String, @@ -28,20 +28,108 @@ struct Finding { detector_version: Option, } -#[wasm_bindgen] -pub fn scan(text: &str) -> Result { - let findings: Vec = datafog_core::scan(text) - .into_iter() - .map(|finding| Finding { +impl From for datafog_core::Finding { + fn from(finding: Finding) -> Self { + Self { entity_type: finding.entity_type, matched_text: finding.matched_text, - byte_range: finding.byte_range.into(), - codepoint_range: finding.codepoint_range.into(), + byte_range: datafog_core::TextRange { + start: finding.byte_range.start, + end: finding.byte_range.end, + }, + codepoint_range: datafog_core::TextRange { + start: finding.codepoint_range.start, + end: finding.codepoint_range.end, + }, confidence: finding.confidence, detector_name: finding.detector_name, detector_version: finding.detector_version, - }) + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Transformation { + finding: Finding, + strategy: &'static str, + replacement: String, + output_byte_range: TextRange, + output_codepoint_range: TextRange, +} + +#[derive(Serialize)] +struct TransformResult { + text: String, + transformations: Vec, +} + +fn finding_from_core(finding: datafog_core::Finding) -> Finding { + Finding { + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: finding.byte_range.into(), + codepoint_range: finding.codepoint_range.into(), + confidence: finding.confidence, + detector_name: finding.detector_name, + detector_version: finding.detector_version, + } +} + +fn strategy_from_js(strategy: &str) -> Result { + match strategy { + "redact" => Ok(datafog_core::TransformationStrategy::Redact), + _ => Err(JsValue::from_str("strategy must be 'redact'")), + } +} + +fn result_to_js(result: datafog_core::TransformResult) -> Result { + let result = TransformResult { + text: result.text, + transformations: result + .transformations + .into_iter() + .map(|transformation| Transformation { + finding: finding_from_core(transformation.finding), + strategy: match transformation.strategy { + datafog_core::TransformationStrategy::Redact => "redact", + }, + replacement: transformation.replacement, + output_byte_range: transformation.output_byte_range.into(), + output_codepoint_range: transformation.output_codepoint_range.into(), + }) + .collect(), + }; + + serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) +} + +#[wasm_bindgen] +pub fn scan(text: &str) -> Result { + let findings: Vec = datafog_core::scan(text) + .into_iter() + .map(finding_from_core) .collect(); serde_wasm_bindgen::to_value(&findings).map_err(|error| JsValue::from_str(&error.to_string())) } + +#[wasm_bindgen] +pub fn transform(text: &str, findings: JsValue, strategy: &str) -> Result { + let findings: Vec = serde_wasm_bindgen::from_value(findings) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let findings = findings + .into_iter() + .map(datafog_core::Finding::from) + .collect::>(); + let result = datafog_core::transform(text, &findings, strategy_from_js(strategy)?) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + result_to_js(result) +} + +#[wasm_bindgen] +pub fn scan_and_transform(text: &str, strategy: &str) -> Result { + let result = datafog_core::scan_and_transform(text, strategy_from_js(strategy)?) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + result_to_js(result) +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 19b4905..3aca1da 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -32,6 +32,79 @@ pub struct Finding { pub detector_version: Option, } +/// A privacy transformation applied to a finding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransformationStrategy { + /// Replace the finding with its unnumbered entity-type placeholder. + Redact, +} + +/// One transformation applied to the source text. +#[derive(Debug, Clone, PartialEq)] +pub struct Transformation { + /// The source finding that was transformed. + pub finding: Finding, + /// Strategy applied to the finding. + pub strategy: TransformationStrategy, + /// Exact replacement inserted into the output text. + pub replacement: String, + /// Range of the replacement in UTF-8 bytes in the output text. + pub output_byte_range: TextRange, + /// Range of the replacement in Unicode code points in the output text. + pub output_codepoint_range: TextRange, +} + +/// Text and audit records produced by a transformation. +#[derive(Debug, Clone, PartialEq)] +pub struct TransformResult { + /// Transformed text. + pub text: String, + /// Applied transformations in source document order. + pub transformations: Vec, +} + +/// Reason a caller-supplied finding is invalid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FindingValidationError { + /// The UTF-8 byte range is empty or reversed. + EmptyOrReversedByteRange, + /// The UTF-8 byte range extends beyond the source text. + ByteRangeOutOfBounds, + /// A UTF-8 byte offset does not fall on a character boundary. + InvalidUtf8Boundary, + /// The Unicode code-point range is empty or reversed. + EmptyOrReversedCodepointRange, + /// The Unicode code-point range extends beyond the source text. + CodepointRangeOutOfBounds, + /// The byte and code-point ranges select different source spans. + InconsistentRanges, + /// `matched_text` differs from the source substring selected by the range. + MatchedTextMismatch, + /// Confidence is non-finite or outside `0.0..=1.0`. + InvalidConfidence, +} + +/// A transformation request could not be completed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TransformError { + /// Index of the invalid finding in the caller-supplied slice. + pub finding_index: usize, + /// Validation failure. + pub kind: FindingValidationError, +} + +impl std::fmt::Display for TransformError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "invalid finding at index {}: {:?}", + self.finding_index, self.kind + ) + } +} + +impl std::error::Error for TransformError {} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] enum Label { Email, @@ -90,6 +163,219 @@ pub fn scan(text: &str) -> Vec { finalize(text, candidates) } +/// Transform caller-supplied findings without scanning implicitly. +pub fn transform( + text: &str, + findings: &[Finding], + strategy: TransformationStrategy, +) -> Result { + for (finding_index, finding) in findings.iter().enumerate() { + if let Err(kind) = validate_finding(text, finding) { + return Err(TransformError { + finding_index, + kind, + }); + } + } + + let mut selected_findings: Vec = Vec::with_capacity(findings.len()); + for finding in findings { + if let Some(existing) = selected_findings + .iter_mut() + .find(|existing| findings_are_duplicates(existing, finding)) + { + if duplicate_preference(finding, existing).is_lt() { + *existing = finding.clone(); + } + } else { + selected_findings.push(finding.clone()); + } + } + selected_findings.sort_by_key(|finding| { + ( + finding.codepoint_range.start, + finding.codepoint_range.end, + finding.entity_type.clone(), + ) + }); + let selected_findings = resolve_overlaps(selected_findings); + + let mut output = String::with_capacity(text.len()); + let mut transformations = Vec::with_capacity(selected_findings.len()); + let mut source_byte_cursor = 0; + + for finding in &selected_findings { + output.push_str(&text[source_byte_cursor..finding.byte_range.start]); + let output_byte_start = output.len(); + let output_codepoint_start = output.chars().count(); + let replacement = match strategy { + TransformationStrategy::Redact => format!("[{}]", finding.entity_type), + }; + output.push_str(&replacement); + + transformations.push(Transformation { + finding: finding.clone(), + strategy, + replacement, + output_byte_range: TextRange { + start: output_byte_start, + end: output.len(), + }, + output_codepoint_range: TextRange { + start: output_codepoint_start, + end: output.chars().count(), + }, + }); + source_byte_cursor = finding.byte_range.end; + } + + output.push_str(&text[source_byte_cursor..]); + Ok(TransformResult { + text: output, + transformations, + }) +} + +/// Scan text and transform the resulting findings in one explicit convenience operation. +pub fn scan_and_transform( + text: &str, + strategy: TransformationStrategy, +) -> Result { + transform(text, &scan(text), strategy) +} + +fn findings_are_duplicates(left: &Finding, right: &Finding) -> bool { + left.entity_type == right.entity_type + && left.matched_text == right.matched_text + && left.byte_range == right.byte_range + && left.codepoint_range == right.codepoint_range +} + +fn duplicate_preference(candidate: &Finding, existing: &Finding) -> std::cmp::Ordering { + if let (Some(candidate_confidence), Some(existing_confidence)) = + (candidate.confidence, existing.confidence) + { + let confidence_order = existing_confidence.total_cmp(&candidate_confidence); + if !confidence_order.is_eq() { + return confidence_order; + } + } + + (&candidate.detector_name, &candidate.detector_version) + .cmp(&(&existing.detector_name, &existing.detector_version)) +} + +fn resolve_overlaps(mut remaining: Vec) -> Vec { + let mut selected = Vec::with_capacity(remaining.len()); + while !remaining.is_empty() { + let mut preferred_index = 0; + for candidate_index in 1..remaining.len() { + if overlap_preference(&remaining[candidate_index], &remaining[preferred_index]).is_lt() + { + preferred_index = candidate_index; + } + } + + let preferred = remaining.remove(preferred_index); + remaining.retain(|finding| !findings_overlap(&preferred, finding)); + selected.push(preferred); + } + + selected.sort_by(|left, right| { + left.byte_range + .start + .cmp(&right.byte_range.start) + .then_with(|| left.byte_range.end.cmp(&right.byte_range.end)) + .then_with(|| left.entity_type.cmp(&right.entity_type)) + }); + selected +} + +fn findings_overlap(left: &Finding, right: &Finding) -> bool { + left.byte_range.start < right.byte_range.end && right.byte_range.start < left.byte_range.end +} + +fn overlap_preference(left: &Finding, right: &Finding) -> std::cmp::Ordering { + let left_contains_right = left.byte_range.start <= right.byte_range.start + && left.byte_range.end >= right.byte_range.end; + let right_contains_left = right.byte_range.start <= left.byte_range.start + && right.byte_range.end >= left.byte_range.end; + match (left_contains_right, right_contains_left) { + (true, false) => return std::cmp::Ordering::Less, + (false, true) => return std::cmp::Ordering::Greater, + _ => {} + } + + let left_length = left.codepoint_range.end - left.codepoint_range.start; + let right_length = right.codepoint_range.end - right.codepoint_range.start; + let length_order = right_length.cmp(&left_length); + if !length_order.is_eq() { + return length_order; + } + + if let (Some(left_confidence), Some(right_confidence)) = (left.confidence, right.confidence) { + let confidence_order = right_confidence.total_cmp(&left_confidence); + if !confidence_order.is_eq() { + return confidence_order; + } + } + + left.codepoint_range + .start + .cmp(&right.codepoint_range.start) + .then_with(|| left.entity_type.cmp(&right.entity_type)) + .then_with(|| left.detector_name.cmp(&right.detector_name)) + .then_with(|| left.detector_version.cmp(&right.detector_version)) +} + +fn validate_finding(text: &str, finding: &Finding) -> Result<(), FindingValidationError> { + if finding.byte_range.start >= finding.byte_range.end { + return Err(FindingValidationError::EmptyOrReversedByteRange); + } + if finding.byte_range.end > text.len() { + return Err(FindingValidationError::ByteRangeOutOfBounds); + } + if !text.is_char_boundary(finding.byte_range.start) + || !text.is_char_boundary(finding.byte_range.end) + { + return Err(FindingValidationError::InvalidUtf8Boundary); + } + if finding.codepoint_range.start >= finding.codepoint_range.end { + return Err(FindingValidationError::EmptyOrReversedCodepointRange); + } + + let Some(codepoint_start_byte) = byte_offset_at_codepoint(text, finding.codepoint_range.start) + else { + return Err(FindingValidationError::CodepointRangeOutOfBounds); + }; + let Some(codepoint_end_byte) = byte_offset_at_codepoint(text, finding.codepoint_range.end) + else { + return Err(FindingValidationError::CodepointRangeOutOfBounds); + }; + if codepoint_start_byte != finding.byte_range.start + || codepoint_end_byte != finding.byte_range.end + { + return Err(FindingValidationError::InconsistentRanges); + } + if text[finding.byte_range.start..finding.byte_range.end] != finding.matched_text { + return Err(FindingValidationError::MatchedTextMismatch); + } + if finding + .confidence + .is_some_and(|confidence| !confidence.is_finite() || !(0.0..=1.0).contains(&confidence)) + { + return Err(FindingValidationError::InvalidConfidence); + } + Ok(()) +} + +fn byte_offset_at_codepoint(text: &str, codepoint_offset: usize) -> Option { + text.char_indices() + .map(|(byte_offset, _)| byte_offset) + .chain(std::iter::once(text.len())) + .nth(codepoint_offset) +} + fn finalize(text: &str, mut candidates: Vec) -> Vec { candidates.sort_by(|left, right| { left.start_byte @@ -735,7 +1021,10 @@ fn detect_ip_address(text: &str, candidates: &mut Vec) { #[cfg(test)] mod tests { - use super::{Finding, TextRange, scan}; + use super::{ + Finding, FindingValidationError, TextRange, TransformError, TransformationStrategy, scan, + scan_and_transform, transform, + }; fn expected_finding( entity_type: &str, @@ -777,6 +1066,25 @@ mod tests { ) } + fn supplied_ascii_finding( + text: &str, + entity_type: &str, + start: usize, + end: usize, + confidence: Option, + detector_name: &str, + ) -> Finding { + Finding { + entity_type: entity_type.to_owned(), + matched_text: text[start..end].to_owned(), + byte_range: TextRange { start, end }, + codepoint_range: TextRange { start, end }, + confidence, + detector_name: detector_name.to_owned(), + detector_version: Some("1".to_owned()), + } + } + #[test] fn empty_input_has_no_entities() { assert!(scan("").is_empty()); @@ -991,4 +1299,255 @@ mod tests { assert!(scan("2001:db8::zzzz").is_empty()); assert!(scan("host192.168.1.10name").is_empty()); } + + #[test] + fn redacts_explicit_findings_and_reports_output_ranges() { + let text = "Contact jane@example.com"; + let findings = scan(text); + + let result = transform(text, &findings, TransformationStrategy::Redact).unwrap(); + + assert_eq!(result.text, "Contact [EMAIL]"); + assert_eq!(result.transformations.len(), 1); + let transformation = &result.transformations[0]; + assert_eq!(transformation.finding, findings[0]); + assert_eq!(transformation.strategy, TransformationStrategy::Redact); + assert_eq!(transformation.replacement, "[EMAIL]"); + assert_eq!( + transformation.output_byte_range, + TextRange { start: 8, end: 15 } + ); + assert_eq!( + transformation.output_codepoint_range, + TextRange { start: 8, end: 15 } + ); + } + + #[test] + fn rejects_a_finding_whose_matched_text_differs_from_the_source() { + let text = "Contact jane@example.com"; + let mut findings = scan(text); + findings[0].matched_text = "other@example.com".to_owned(); + + assert_eq!( + transform(text, &findings, TransformationStrategy::Redact), + Err(TransformError { + finding_index: 0, + kind: FindingValidationError::MatchedTextMismatch, + }) + ); + } + + #[test] + fn rejects_malformed_ranges_and_confidence() { + let text = "👋 jane@example.com"; + let original = scan(text).remove(0); + + let cases = [ + ( + Finding { + byte_range: TextRange { start: 5, end: 5 }, + ..original.clone() + }, + FindingValidationError::EmptyOrReversedByteRange, + ), + ( + Finding { + byte_range: TextRange { start: 5, end: 99 }, + ..original.clone() + }, + FindingValidationError::ByteRangeOutOfBounds, + ), + ( + Finding { + byte_range: TextRange { start: 1, end: 21 }, + ..original.clone() + }, + FindingValidationError::InvalidUtf8Boundary, + ), + ( + Finding { + codepoint_range: TextRange { start: 2, end: 2 }, + ..original.clone() + }, + FindingValidationError::EmptyOrReversedCodepointRange, + ), + ( + Finding { + codepoint_range: TextRange { start: 2, end: 99 }, + ..original.clone() + }, + FindingValidationError::CodepointRangeOutOfBounds, + ), + ( + Finding { + codepoint_range: TextRange { start: 1, end: 17 }, + ..original.clone() + }, + FindingValidationError::InconsistentRanges, + ), + ( + Finding { + confidence: Some(f32::NAN), + ..original.clone() + }, + FindingValidationError::InvalidConfidence, + ), + ( + Finding { + confidence: Some(-0.1), + ..original.clone() + }, + FindingValidationError::InvalidConfidence, + ), + ( + Finding { + confidence: Some(1.1), + ..original.clone() + }, + FindingValidationError::InvalidConfidence, + ), + ]; + + for (finding, expected_kind) in cases { + assert_eq!( + transform(text, &[finding], TransformationStrategy::Redact), + Err(TransformError { + finding_index: 0, + kind: expected_kind, + }) + ); + } + } + + #[test] + fn collapses_duplicate_findings_and_retains_higher_confidence() { + let text = "Email jane@example.com"; + let mut lower_confidence = scan(text).remove(0); + lower_confidence.confidence = Some(0.7); + lower_confidence.detector_name = "z-detector".to_owned(); + let mut higher_confidence = lower_confidence.clone(); + higher_confidence.confidence = Some(0.9); + higher_confidence.detector_name = "a-detector".to_owned(); + + let result = transform( + text, + &[lower_confidence, higher_confidence.clone()], + TransformationStrategy::Redact, + ) + .unwrap(); + + assert_eq!(result.text, "Email [EMAIL]"); + assert_eq!(result.transformations.len(), 1); + assert_eq!(result.transformations[0].finding, higher_confidence); + } + + #[test] + fn containing_overlap_wins_even_when_the_inner_finding_has_higher_confidence() { + let text = "Acme Corporation announced"; + let outer = supplied_ascii_finding( + text, + "ORGANIZATION", + 0, + 16, + Some(0.6), + "organization-detector", + ); + let inner = supplied_ascii_finding(text, "PERSON", 0, 4, Some(0.99), "person-detector"); + + let result = transform( + text, + &[inner, outer.clone()], + TransformationStrategy::Redact, + ) + .unwrap(); + + assert_eq!(result.text, "[ORGANIZATION] announced"); + assert_eq!(result.transformations.len(), 1); + assert_eq!(result.transformations[0].finding, outer); + } + + #[test] + fn scan_and_transform_redacts_unicode_input_with_exact_output_ranges() { + let text = "👋 jane@example.com and jane@example.com"; + + let result = scan_and_transform(text, TransformationStrategy::Redact).unwrap(); + + assert_eq!(result.text, "👋 [EMAIL] and [EMAIL]"); + assert_eq!(result.transformations.len(), 2); + assert_eq!( + result.transformations[0].output_byte_range, + TextRange { start: 5, end: 12 } + ); + assert_eq!( + result.transformations[0].output_codepoint_range, + TextRange { start: 2, end: 9 } + ); + assert_eq!( + &result.text[result.transformations[1].output_byte_range.start + ..result.transformations[1].output_byte_range.end], + "[EMAIL]" + ); + assert_eq!(result.transformations[0].replacement, "[EMAIL]"); + assert_eq!(result.transformations[1].replacement, "[EMAIL]"); + } + + #[test] + fn equal_length_overlaps_use_confidence_when_both_findings_provide_it() { + let text = "123456789"; + let lower = supplied_ascii_finding(text, "ALPHA", 0, 9, Some(0.7), "a"); + let higher = supplied_ascii_finding(text, "ZETA", 0, 9, Some(0.9), "z"); + + let result = transform( + text, + &[lower, higher.clone()], + TransformationStrategy::Redact, + ) + .unwrap(); + + assert_eq!(result.text, "[ZETA]"); + assert_eq!(result.transformations[0].finding, higher); + } + + #[test] + fn missing_confidence_does_not_rank_as_zero() { + let text = "123456789"; + let unscored = supplied_ascii_finding(text, "ALPHA", 0, 9, None, "z"); + let scored = supplied_ascii_finding(text, "ZETA", 0, 9, Some(0.99), "a"); + + let result = transform( + text, + &[scored, unscored.clone()], + TransformationStrategy::Redact, + ) + .unwrap(); + + assert_eq!(result.text, "[ALPHA]"); + assert_eq!(result.transformations[0].finding, unscored); + } + + #[test] + fn equal_partial_overlaps_prefer_the_earlier_source_position() { + let text = "abcdef"; + let earlier = supplied_ascii_finding(text, "ZETA", 0, 4, None, "z"); + let later = supplied_ascii_finding(text, "ALPHA", 2, 6, None, "a"); + + let result = transform( + text, + &[later, earlier.clone()], + TransformationStrategy::Redact, + ) + .unwrap(); + + assert_eq!(result.text, "[ZETA]ef"); + assert_eq!(result.transformations[0].finding, earlier); + } + + #[test] + fn empty_findings_leave_text_unchanged() { + let result = transform("plain text", &[], TransformationStrategy::Redact).unwrap(); + + assert_eq!(result.text, "plain text"); + assert!(result.transformations.is_empty()); + } } diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index 974ecf0..578196b 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -12,6 +12,7 @@ The engine is organized around three operations: ```text scan(text) -> findings transform(...) -> transformed text and transformation records +scan_and_transform(...) -> explicit scan-then-transform convenience restore(...) -> authorized restoration of reversible tokens ``` @@ -20,6 +21,9 @@ restore(...) -> authorized restoration of reversible tokens ### Operation taxonomy Canonical core operations are `scan`, `transform`, and `restore`. +`transform` always requires caller-supplied findings and never scans +implicitly. `scan_and_transform` is the explicitly named convenience operation +for callers that want Core to perform both steps. Canonical transformation strategies are: @@ -92,20 +96,22 @@ construction. changes. Exact duplicates have the same entity type, source range, and matched text and -collapse into one finding. If duplicate detector results differ, the result -with higher confidence is retained; remaining ties use stable detector-name -ordering. +collapse into one finding. If both duplicate detector results have confidence, +the result with higher confidence is retained; remaining ties use stable +detector provenance ordering. Overlapping findings are ranked by: -1. longer source span; -2. higher entity-type priority; -3. higher confidence; -4. earlier source position; and -5. stable lexical ordering. +1. a containing span over a span contained within it; +2. longer Unicode code-point span; +3. higher confidence when both findings provide confidence; +4. earlier source position; +5. lexical entity-type ordering; and +6. lexical detector-name and detector-version ordering. The selected non-overlapping findings are returned in document order. Entity -priority has a core default and may later be made configurable. +types have equal priority by default. A later transformation configuration may +provide explicit domain-specific priorities. ### Transformation result @@ -122,21 +128,25 @@ Transformation { finding strategy replacement - output_range + output_byte_range + output_codepoint_range } ``` Transformation records are ordered by source document position and include -only transformations that were actually applied. The finding already supplies -the original value and source range, so the canonical payload does not contain -a second original-to-replacement mapping. +only transformations that were actually applied. Output ranges are zero-based, +end-exclusive, refer to the transformed text, and select exactly `replacement`. +The finding already supplies the original value and source ranges, so the +canonical payload does not contain a second original-to-replacement mapping. A mapping view may be offered as an explicit convenience API. Sensitive original values are excluded from default debug and log output. ### Security meaning of strategies -- `redact` creates typed, document-local placeholders. +- `redact` replaces every selected finding with the unnumbered placeholder + `[ENTITY_TYPE]`. Repeated occurrences intentionally receive the same + type-only placeholder; this makes no identity or equality claim. - `mask` hides all or a configured part of a value. - `hash` is a compatibility fingerprint with explicitly documented leakage and must not be presented as secure pseudonymization. diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 02801ff..9858c72 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -11,6 +11,7 @@ The target model is: ```text scan(text) -> findings transform(...) -> transformed text and transformation records +scan_and_transform(...) -> explicit scan-then-transform convenience restore(...) -> authorized restoration of reversible tokens ``` @@ -49,11 +50,18 @@ round-trip fixtures prove every public range selects the reported matched text. ## Slice 2: Transformation framework and redaction +**Status: complete** + +- Require explicit findings in `transform`; never scan implicitly. +- Provide `scan_and_transform` as the explicit convenience operation. - Validate caller-supplied findings strictly. - Deduplicate and resolve overlaps once in shared code. - Apply replacements without invalidating subsequent source ranges. -- Implement typed, document-local redaction placeholders. -- Return ordered transformation records and output ranges. +- Implement unnumbered `[ENTITY_TYPE]` redaction placeholders. +- Return ordered transformation records with explicit output byte and + code-point ranges. +- Keep entity types equal by default and use deterministic structural, + confidence, position, and lexical tie-breaking. **Proof:** normal, repeated, duplicate, overlapping, nested, empty, malformed, and Unicode cases satisfy ADR 001. diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index e2e2322..6e4f596 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -30,7 +30,7 @@ function writeConsumerTest() { import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { scan } from "@datafog/node"; +import { scan, scanAndTransform, transform } from "@datafog/node"; const fixturesDirectory = process.argv[2]; @@ -83,21 +83,47 @@ const emojiFinding = scan("👋 jane@example.com")[0]; assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); -console.log("Installed @datafog/node package matches fixtures and the Finding contract."); +const transformText = "👋 jane@example.com and jane@example.com"; +const explicit = transform(transformText, scan(transformText), "redact"); +const convenience = scanAndTransform(transformText, "redact"); +assert.deepEqual(explicit, convenience); +assert.equal(explicit.text, "👋 [EMAIL] and [EMAIL]"); +assert.equal(explicit.transformations.length, 2); +assert.equal(explicit.transformations[0].replacement, "[EMAIL]"); +assert.deepEqual(explicit.transformations[0].outputByteRange, { start: 5, end: 12 }); +assert.deepEqual(explicit.transformations[0].outputCodepointRange, { start: 2, end: 9 }); +assert.throws( + () => transform(transformText, [{ ...scan(transformText)[0], confidence: 2 }], "redact"), + /InvalidConfidence/, +); + +console.log("Installed @datafog/node package matches fixtures and transform contracts."); `.trimStart(), ); writeFileSync( path.join(temporaryDirectory, "type-smoke.ts"), ` -import { scan, type EntityType, type Finding, type TextRange } from "@datafog/node"; +import { + scan, + scanAndTransform, + transform, + type EntityType, + type Finding, + type TextRange, + type TransformResult, +} from "@datafog/node"; const findings: Finding[] = scan("Email jane@example.com"); const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; +const explicit: TransformResult = transform("Email jane@example.com", findings, "redact"); +const convenience: TransformResult = scanAndTransform("Email jane@example.com", "redact"); void entityType; void range; +void explicit; +void convenience; `.trimStart(), ); diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index cc91e61..59a63a8 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -78,16 +78,36 @@ function writeConsumerFiles() { writeFileSync( path.join(temporaryDirectory, "type-smoke.ts"), ` -import { init, scan, type EntityType, type Finding, type TextRange } from "@datafog/wasm"; +import { + init, + scan, + scanAndTransform, + transform, + type EntityType, + type Finding, + type TextRange, + type TransformResult, +} from "@datafog/wasm"; const ready: Promise = init(); const findings: Finding[] = scan("Email jane@example.com"); const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; +const transformed: TransformResult = transform( + "Email jane@example.com", + findings, + "redact", +); +const scannedAndTransformed: TransformResult = scanAndTransform( + "Email jane@example.com", + "redact", +); void ready; void entityType; void range; +void transformed; +void scannedAndTransformed; `.trimStart(), ); @@ -161,7 +181,9 @@ try { await page.goto(serverInfo.url); await page.evaluate(async () => { - const { init, scan } = await import("/node_modules/@datafog/wasm/index.js"); + const { init, scan, scanAndTransform, transform } = await import( + "/node_modules/@datafog/wasm/index.js" + ); function expectThrows(callback, name) { try { @@ -236,6 +258,35 @@ try { ) { throw new Error("Unicode ranges do not use the documented coordinate systems"); } + + const text = "👋 jane@example.com and jane@example.com"; + const findings = scan(text); + const explicit = transform(text, findings, "redact"); + const convenient = scanAndTransform(text, "redact"); + if (JSON.stringify(explicit) !== JSON.stringify(convenient)) { + throw new Error("explicit and convenience transforms differ"); + } + if (explicit.text !== "👋 [EMAIL] and [EMAIL]") { + throw new Error(`unexpected transformed text: ${explicit.text}`); + } + if ( + explicit.transformations.length !== 2 || + explicit.transformations.some( + (record) => + record.strategy !== "redact" || + record.replacement !== "[EMAIL]" || + Array.from(explicit.text) + .slice(record.outputCodepointRange.start, record.outputCodepointRange.end) + .join("") !== record.replacement, + ) + ) { + throw new Error("transformation records do not select their replacements"); + } + + expectThrows( + () => transform(text, [{ ...findings[0], confidence: 2 }], "redact"), + "Error", + ); }); console.log("Installed @datafog/wasm package matches fixtures and the Finding contract.");