diff --git a/Cargo.lock b/Cargo.lock index 14795d3..4d659b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,6 +59,7 @@ version = "0.1.0" dependencies = [ "datafog-core", "pyo3", + "serde_json", ] [[package]] @@ -69,6 +70,7 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "serde_json", ] [[package]] @@ -78,6 +80,7 @@ dependencies = [ "datafog-core", "serde", "serde-wasm-bindgen", + "serde_json", "wasm-bindgen", ] @@ -228,6 +231,8 @@ dependencies = [ "napi-sys", "nohash-hasher", "rustc-hash", + "serde", + "serde_json", ] [[package]] diff --git a/README.md b/README.md index 69a3edd..cbaaea5 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,27 @@ span. `transform` requires explicit findings; `scan_and_transform` (or convenience. Results include the transformed text and an ordered record for every applied replacement, including its output byte and code-point ranges. -Object-oriented bindings use a discriminated configuration: +Transformation calls require an envelope with a default strategy. It can also +select entity types, override the strategy per entity, and exempt exact or +full-match regex values: -```text -{ strategy: "redact" } -{ strategy: "remove" } -{ strategy: "mask", character: "*", reveal: { direction: "last", count: 4 } } +```js +{ + default: { strategy: "redact" }, + entities: ["EMAIL", "PHONE"], + overrides: { + PHONE: { strategy: "mask", reveal: { direction: "last", count: 4 } }, + }, + allow: { + exact: { EMAIL: ["support@example.com"] }, + regex: { EMAIL: [{ pattern: ".+@example\\.org" }] }, + }, +} ``` +`scan_and_transform` uses `{ scan?: { locale?: string }, transform: ... }` so +detection settings remain separate from transformation policy. + ## Packages | Runtime | Distribution | Import | Status | @@ -47,7 +60,10 @@ cargo add datafog-core ``` ```rust -use datafog_core::{scan, scan_and_transform, TransformationStrategy}; +use datafog_core::{ + scan, scan_and_transform, ScanAndTransformConfig, TransformationConfig, + TransformationStrategy, +}; let findings = scan("Email jane@example.com"); assert_eq!(findings[0].entity_type, "EMAIL"); @@ -56,7 +72,9 @@ assert_eq!(findings[0].byte_range.start, 6); let result = scan_and_transform( "Email jane@example.com", - TransformationStrategy::Redact, + &ScanAndTransformConfig::new(TransformationConfig::new( + TransformationStrategy::Redact, + )), ).unwrap(); assert_eq!(result.text, "Email [EMAIL]"); ``` @@ -75,12 +93,22 @@ print(findings[0].entity_type) # EMAIL print(findings[0].matched_text) # jane@example.com print(findings[0].byte_range.start) # 6 -result = scan_and_transform("Email jane@example.com", {"strategy": "redact"}) +result = scan_and_transform( + "Email jane@example.com", + {"transform": {"default": {"strategy": "redact"}}}, +) assert result.text == "Email [EMAIL]" masked = scan_and_transform( "Email jane@example.com", - {"strategy": "mask", "reveal": {"direction": "last", "count": 4}}, + { + "transform": { + "default": { + "strategy": "mask", + "reveal": {"direction": "last", "count": 4}, + } + } + }, ) assert masked.text == "Email ************.com" ``` @@ -94,7 +122,9 @@ import { scan, scanAndTransform } from "@datafog/node"; console.log(scan("Email jane@example.com")); console.log( - scanAndTransform("Email jane@example.com", { strategy: "redact" }).text, + scanAndTransform("Email jane@example.com", { + transform: { default: { strategy: "redact" } }, + }).text, ); ``` @@ -110,7 +140,9 @@ import { init, scan, scanAndTransform } from "@datafog/wasm"; await init(); console.log(scan("Email jane@example.com")); console.log( - scanAndTransform("Email jane@example.com", { strategy: "redact" }).text, + scanAndTransform("Email jane@example.com", { + transform: { default: { strategy: "redact" } }, + }).text, ); ``` diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 77a8171..1e7346f 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -12,8 +12,9 @@ crate-type = ["cdylib"] [dependencies] datafog-core = { path = "../../crates/core" } -napi = "=3.12.2" +napi = { version = "=3.12.2", features = ["serde-json"] } napi-derive = "=3.6.3" +serde_json = "1" [build-dependencies] napi-build = "=2.4.1" diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index b816875..f413c4c 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -8,7 +8,7 @@ export interface MaskRevealConfig { readonly count: number; } -export type TransformationConfig = +export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } | { @@ -16,3 +16,41 @@ export type TransformationConfig = readonly character?: string; readonly reveal?: MaskRevealConfig; }; + +export interface RegexAllowRule { + readonly pattern: string; + readonly case_sensitive?: boolean; +} + +export interface AllowConfig { + readonly exact?: Readonly>; + readonly regex?: Readonly>; +} + +export interface TransformationConfig { + readonly default: TransformationStrategyConfig; + readonly entities?: readonly EntityType[]; + readonly overrides?: Readonly>; + readonly allow?: AllowConfig; +} + +export interface ScanConfig { + readonly locale?: string; +} + +export interface ScanAndTransformConfig { + readonly scan?: ScanConfig; + readonly transform: TransformationConfig; +} + +export type DataFogErrorCode = + | "invalid_configuration" + | "invalid_finding" + | "internal_error"; + +export declare class DataFogError extends Error { + readonly code: DataFogErrorCode; + readonly reason?: string; + readonly path?: string; + readonly findingIndex?: number; +} diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 3e8d8f6..a47594a 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -8,7 +8,7 @@ export interface MaskRevealConfig { readonly count: number; } -export type TransformationConfig = +export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } | { @@ -16,6 +16,44 @@ export type TransformationConfig = readonly character?: string; readonly reveal?: MaskRevealConfig; }; + +export interface RegexAllowRule { + readonly pattern: string; + readonly case_sensitive?: boolean; +} + +export interface AllowConfig { + readonly exact?: Readonly>; + readonly regex?: Readonly>; +} + +export interface TransformationConfig { + readonly default: TransformationStrategyConfig; + readonly entities?: readonly EntityType[]; + readonly overrides?: Readonly>; + readonly allow?: AllowConfig; +} + +export interface ScanConfig { + readonly locale?: string; +} + +export interface ScanAndTransformConfig { + readonly scan?: ScanConfig; + readonly transform: TransformationConfig; +} + +export type DataFogErrorCode = + | "invalid_configuration" + | "invalid_finding" + | "internal_error"; + +export declare class DataFogError extends Error { + readonly code: DataFogErrorCode; + readonly reason?: string; + readonly path?: string; + readonly findingIndex?: number; +} export interface Finding { readonly entityType: EntityType readonly matchedText: string @@ -26,22 +64,11 @@ export interface Finding { readonly detectorVersion?: string } -export interface NativeMaskRevealConfig { - direction: string - count: number -} - -export interface NativeTransformationConfig { - strategy: TransformationStrategy - character?: string - reveal?: NativeMaskRevealConfig -} - /** Scan text for supported PII findings. */ -export declare function scan(text: string): Array +export declare function scan(text: string, config?: ScanConfig | undefined): Array /** Scan text and transform the detected findings. */ -export declare function scanAndTransform(text: string, config: TransformationConfig): TransformResult +export declare function scanAndTransform(text: string, config: ScanAndTransformConfig): TransformResult export interface TextRange { readonly start: number diff --git a/bindings/node/index.js b/bindings/node/index.js index 923b942..b82adae 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -4,70 +4,51 @@ import { transform as nativeTransform, } from "./native.js"; -export function scan(text) { - if (typeof text !== "string") { - throw new TypeError("scan text must be a string"); +export class DataFogError extends Error { + constructor({ code, reason, message, path, findingIndex }) { + super(message); + this.name = "DataFogError"; + this.code = code; + this.reason = reason; + this.path = path; + this.findingIndex = findingIndex; } - - return nativeScan(text); } -function validateConfig(config) { - if (typeof config !== "object" || config === null || Array.isArray(config)) { - throw new TypeError("transformation configuration must be an object"); - } - if (!["redact", "mask", "remove"].includes(config.strategy)) { - throw new TypeError("strategy must be 'redact', 'mask', or 'remove'"); - } - - const allowed = - config.strategy === "mask" - ? new Set(["strategy", "character", "reveal"]) - : new Set(["strategy"]); - for (const key of Object.keys(config)) { - if (!allowed.has(key)) { - throw new TypeError(`unexpected configuration field: ${key}`); +function normalizeError(error, fallbackCode) { + if (error instanceof DataFogError) return error; + try { + const details = JSON.parse(error?.message ?? ""); + if (typeof details.code === "string" && typeof details.message === "string") { + return new DataFogError(details); } + } catch { + // Native conversion errors use the operation-specific fallback below. } + return new DataFogError({ + code: fallbackCode, + reason: fallbackCode === "invalid_configuration" ? "invalid_type" : undefined, + message: + fallbackCode === "invalid_configuration" + ? "request configuration could not be decoded" + : "the native operation failed unexpectedly", + path: fallbackCode === "invalid_configuration" ? "" : undefined, + }); +} - if (config.strategy === "mask") { - if (config.character !== undefined) { - if ( - typeof config.character !== "string" || - Array.from(config.character).length !== 1 || - /[\p{White_Space}\p{Cc}]/u.test(config.character) - ) { - throw new TypeError( - "mask character must be one non-whitespace, non-control code point", - ); - } - } - if (config.reveal !== undefined) { - if ( - typeof config.reveal !== "object" || - config.reveal === null || - Array.isArray(config.reveal) - ) { - throw new TypeError("mask reveal configuration must be an object"); - } - for (const key of Object.keys(config.reveal)) { - if (key !== "direction" && key !== "count") { - throw new TypeError(`unexpected reveal field: ${key}`); - } - } - if (!["first", "last"].includes(config.reveal.direction)) { - throw new TypeError("reveal direction must be 'first' or 'last'"); - } - if ( - !Number.isSafeInteger(config.reveal.count) || - config.reveal.count < 0 - ) { - throw new TypeError("reveal count must be a non-negative safe integer"); - } - } +export function scan(text, config) { + if (typeof text !== "string") { + throw new TypeError("scan text must be a string"); } - return config; + try { + return nativeScan(text, config); + } catch (error) { + throw normalizeError( + error, + config === undefined ? "internal_error" : "invalid_configuration", + ); + } } export function transform(text, findings, config) { @@ -78,7 +59,11 @@ export function transform(text, findings, config) { throw new TypeError("transform findings must be an array"); } - return nativeTransform(text, findings, validateConfig(config)); + try { + return nativeTransform(text, findings, config); + } catch (error) { + throw normalizeError(error, "invalid_configuration"); + } } export function scanAndTransform(text, config) { @@ -86,5 +71,9 @@ export function scanAndTransform(text, config) { throw new TypeError("scanAndTransform text must be a string"); } - return nativeScanAndTransform(text, validateConfig(config)); + try { + return nativeScanAndTransform(text, config); + } catch (error) { + throw normalizeError(error, "invalid_configuration"); + } } diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 9bc1e89..ac07a10 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -1,6 +1,6 @@ //! Node binding for datafog-core. -use napi::{Error, Status}; +use napi::{Env, Error, Status, Unknown}; use napi_derive::napi; #[napi(object)] @@ -36,20 +36,6 @@ pub struct Finding { pub detector_version: Option, } -#[napi(object)] -pub struct NativeMaskRevealConfig { - pub direction: String, - pub count: f64, -} - -#[napi(object)] -pub struct NativeTransformationConfig { - #[napi(ts_type = "TransformationStrategy")] - pub strategy: String, - pub character: Option, - pub reveal: Option, -} - #[napi(object, object_from_js = false)] pub struct Transformation { #[napi(readonly)] @@ -123,72 +109,19 @@ fn core_finding(finding: Finding) -> datafog_core::Finding { } } -fn core_strategy( - config: NativeTransformationConfig, -) -> napi::Result { - match config.strategy.as_str() { - "redact" if config.character.is_none() && config.reveal.is_none() => { - Ok(datafog_core::TransformationStrategy::Redact) - } - "remove" if config.character.is_none() && config.reveal.is_none() => { - Ok(datafog_core::TransformationStrategy::Remove) - } - "mask" => { - let character = config.character.unwrap_or_else(|| "*".to_owned()); - let mut characters = character.chars(); - let character = characters - .next() - .filter(|_| characters.next().is_none()) - .ok_or_else(|| { - Error::new( - Status::InvalidArg, - "mask character must contain exactly one code point", - ) - })?; - let reveal = match config.reveal { - None => datafog_core::MaskReveal::None, - Some(reveal) => { - if !reveal.count.is_finite() - || reveal.count < 0.0 - || reveal.count.fract() != 0.0 - || reveal.count > 9_007_199_254_740_991.0 - { - return Err(Error::new( - Status::InvalidArg, - "reveal count must be a non-negative safe integer", - )); - } - let count = reveal.count as usize; - match reveal.direction.as_str() { - "first" => datafog_core::MaskReveal::First(count), - "last" => datafog_core::MaskReveal::Last(count), - _ => { - return Err(Error::new( - Status::InvalidArg, - "reveal direction must be 'first' or 'last'", - )); - } - } - } - }; - datafog_core::MaskConfig::new(character, reveal) - .map(datafog_core::TransformationStrategy::Mask) - .map_err(|_| { - Error::new( - Status::InvalidArg, - "mask character must not be whitespace or a control character", - ) - }) - } - "redact" | "remove" => Err(Error::new( - Status::InvalidArg, - "redact and remove do not accept mask configuration", - )), - _ => Err(Error::new( - Status::InvalidArg, - "strategy must be 'redact', 'mask', or 'remove'", - )), - } +fn js_privacy_error(error: datafog_core::PrivacyError) -> Error { + let payload = serde_json::json!({ + "code": error.code().as_str(), + "reason": error.reason().map(datafog_core::PrivacyErrorReason::as_str), + "message": error.to_string(), + "path": error.path(), + "findingIndex": error.finding_index(), + }); + let status = match error.code() { + datafog_core::PrivacyErrorCode::InternalError => Status::GenericFailure, + _ => Status::InvalidArg, + }; + Error::new(status, payload.to_string()) } fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result { @@ -216,8 +149,18 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result napi::Result> { - datafog_core::scan(&text) +pub fn scan( + env: Env, + text: String, + #[napi(ts_arg_type = "ScanConfig | undefined")] config: Option>, +) -> napi::Result> { + let config = if let Some(config) = config { + let config: serde_json::Value = env.from_js_value(config)?; + datafog_core::parse_scan_config(&config).map_err(js_privacy_error)? + } else { + datafog_core::ScanConfig::default() + }; + datafog_core::scan_with_config(&text, &config) .into_iter() .map(js_finding) .collect() @@ -226,23 +169,30 @@ pub fn scan(text: String) -> napi::Result> { /// Transform explicit findings without scanning implicitly. #[napi(strict, catch_unwind)] pub fn transform( + env: Env, text: String, findings: Vec, - #[napi(ts_arg_type = "TransformationConfig")] config: NativeTransformationConfig, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, ) -> napi::Result { + let config: serde_json::Value = env.from_js_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?; let findings = findings.into_iter().map(core_finding).collect::>(); - datafog_core::transform(&text, &findings, core_strategy(config)?) - .map_err(|error| Error::new(Status::InvalidArg, error.to_string())) + datafog_core::transform(&text, &findings, &config) + .map_err(js_privacy_error) .and_then(js_transform_result) } /// Scan text and transform the detected findings. #[napi(strict, catch_unwind)] pub fn scan_and_transform( + env: Env, text: String, - #[napi(ts_arg_type = "TransformationConfig")] config: NativeTransformationConfig, + #[napi(ts_arg_type = "ScanAndTransformConfig")] config: Unknown<'_>, ) -> napi::Result { - datafog_core::scan_and_transform(&text, core_strategy(config)?) - .map_err(|error| Error::new(Status::GenericFailure, error.to_string())) + let config: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_scan_and_transform_config(&config).map_err(js_privacy_error)?; + datafog_core::scan_and_transform(&text, &config) + .map_err(js_privacy_error) .and_then(js_transform_result) } diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index db7fa63..f067f55 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -14,3 +14,4 @@ crate-type = ["cdylib"] [dependencies] datafog-core = { path = "../../crates/core" } pyo3 = { version = "0.29", features = ["abi3-py310"] } +serde_json = "1" diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 957dcf1..f8835f7 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,7 +1,12 @@ use ::datafog_core as core; +use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyAny, PyBool, PyDict}; +use pyo3::types::{PyAny, PyBool, PyDict, PyList, PyTuple}; + +create_exception!(datafog_core, DataFogConfigurationError, PyValueError); +create_exception!(datafog_core, DataFogFindingError, PyValueError); +create_exception!(datafog_core, DataFogInternalError, PyRuntimeError); /// A zero-based, end-exclusive text range. #[pyclass(frozen, skip_from_py_object)] @@ -241,108 +246,123 @@ impl TransformResult { } } -fn validate_config_keys(config: &Bound<'_, PyDict>, allowed: &[&str]) -> PyResult<()> { - for (key, _) in config.iter() { - let key = key - .extract::() - .map_err(|_| PyValueError::new_err("configuration keys must be strings"))?; - if !allowed.contains(&key.as_str()) { - return Err(PyValueError::new_err(format!( - "unexpected configuration field: {key}" - ))); - } - } - Ok(()) +fn configuration_conversion_error(py: Python<'_>, path: &str, message: &str) -> PyErr { + let exception = PyErr::new::(message.to_owned()); + let value = exception.value(py); + let _ = value.setattr("code", "invalid_configuration"); + let _ = value.setattr("reason", "invalid_type"); + let _ = value.setattr("path", path); + let _ = value.setattr("finding_index", py.None()); + exception } -fn required_item<'py>(config: &Bound<'py, PyDict>, key: &str) -> PyResult> { - config - .get_item(key)? - .ok_or_else(|| PyValueError::new_err(format!("missing configuration field: {key}"))) +fn internal_error(py: Python<'_>, message: &str) -> PyErr { + let exception = PyErr::new::(message.to_owned()); + let value = exception.value(py); + let _ = value.setattr("code", "internal_error"); + let _ = value.setattr("reason", py.None()); + let _ = value.setattr("path", py.None()); + let _ = value.setattr("finding_index", py.None()); + exception } -fn parse_strategy(config: &Bound<'_, PyAny>) -> PyResult { - let config = config - .cast::() - .map_err(|_| PyValueError::new_err("strategy configuration must be a dict"))?; - let strategy = required_item(config, "strategy")? - .extract::() - .map_err(|_| PyValueError::new_err("strategy must be a string"))?; - - match strategy.as_str() { - "redact" => { - validate_config_keys(config, &["strategy"])?; - Ok(core::TransformationStrategy::Redact) +fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>, path: &str) -> PyResult { + if value.is_none() { + return Ok(serde_json::Value::Null); + } + if value.is_instance_of::() { + return value.extract::().map(serde_json::Value::Bool); + } + if let Ok(value) = value.extract::() { + return Ok(serde_json::Value::String(value)); + } + if let Ok(value) = value.extract::() { + return Ok(serde_json::Value::Number(value.into())); + } + if let Ok(value) = value.extract::() { + return Ok(serde_json::Value::Number(value.into())); + } + if let Ok(value) = value.extract::() { + return serde_json::Number::from_f64(value) + .map(serde_json::Value::Number) + .ok_or_else(|| { + configuration_conversion_error(py, path, "configuration number must be finite") + }); + } + if let Ok(dictionary) = value.cast::() { + let mut object = serde_json::Map::new(); + for (key, value) in dictionary.iter() { + let key = key.extract::().map_err(|_| { + configuration_conversion_error(py, path, "configuration keys must be strings") + })?; + let child_path = format!("{path}/{}", key.replace('~', "~0").replace('/', "~1")); + object.insert(key, py_to_json(py, &value, &child_path)?); } - "remove" => { - validate_config_keys(config, &["strategy"])?; - Ok(core::TransformationStrategy::Remove) + return Ok(serde_json::Value::Object(object)); + } + if let Ok(list) = value.cast::() { + return list + .iter() + .enumerate() + .map(|(index, value)| py_to_json(py, &value, &format!("{path}/{index}"))) + .collect::>>() + .map(serde_json::Value::Array); + } + if let Ok(tuple) = value.cast::() { + return tuple + .iter() + .enumerate() + .map(|(index, value)| py_to_json(py, &value, &format!("{path}/{index}"))) + .collect::>>() + .map(serde_json::Value::Array); + } + Err(configuration_conversion_error( + py, + path, + "configuration values must be JSON-compatible", + )) +} + +fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr { + let exception = match error.code() { + core::PrivacyErrorCode::InvalidConfiguration => { + PyErr::new::(error.to_string()) } - "mask" => { - validate_config_keys(config, &["strategy", "character", "reveal"])?; - let character = config - .get_item("character")? - .map(|value| value.extract::()) - .transpose() - .map_err(|_| PyValueError::new_err("mask character must be a string"))? - .unwrap_or_else(|| "*".to_owned()); - let mut characters = character.chars(); - let character = characters - .next() - .filter(|_| characters.next().is_none()) - .ok_or_else(|| { - PyValueError::new_err("mask character must contain exactly one code point") - })?; - - let reveal = match config.get_item("reveal")? { - None => core::MaskReveal::None, - Some(reveal) => { - let reveal = reveal.cast::().map_err(|_| { - PyValueError::new_err("mask reveal configuration must be a dict") - })?; - validate_config_keys(reveal, &["direction", "count"])?; - let direction = required_item(reveal, "direction")? - .extract::() - .map_err(|_| PyValueError::new_err("reveal direction must be a string"))?; - let count = required_item(reveal, "count")?; - if count.is_instance_of::() { - return Err(PyValueError::new_err( - "reveal count must be a non-negative integer", - )); - } - let count = count.extract::().map_err(|_| { - PyValueError::new_err("reveal count must be a non-negative integer") - })?; - match direction.as_str() { - "first" => core::MaskReveal::First(count), - "last" => core::MaskReveal::Last(count), - _ => { - return Err(PyValueError::new_err( - "reveal direction must be 'first' or 'last'", - )); - } - } - } - }; - core::MaskConfig::new(character, reveal) - .map(core::TransformationStrategy::Mask) - .map_err(|_| { - PyValueError::new_err( - "mask character must not be whitespace or a control character", - ) - }) + core::PrivacyErrorCode::InvalidFinding => { + PyErr::new::(error.to_string()) } - _ => Err(PyValueError::new_err( - "strategy must be 'redact', 'mask', or 'remove'", - )), - } + core::PrivacyErrorCode::InternalError => { + PyErr::new::(error.to_string()) + } + }; + let value = exception.value(py); + let _ = value.setattr("code", error.code().as_str()); + let _ = value.setattr( + "reason", + error.reason().map(core::PrivacyErrorReason::as_str), + ); + let _ = value.setattr("path", error.path()); + let _ = value.setattr("finding_index", error.finding_index()); + exception } /// Scan text for supported PII findings. #[pyfunction] -fn scan(text: &str) -> PyResult> { - std::panic::catch_unwind(|| core::scan(text).into_iter().map(Finding::from).collect()) - .map_err(|_| PyRuntimeError::new_err("unexpected Rust scan failure")) +#[pyo3(signature = (text, config=None))] +fn scan(py: Python<'_>, text: &str, config: Option<&Bound<'_, PyAny>>) -> PyResult> { + let config = if let Some(config) = config { + let config = py_to_json(py, config, "")?; + core::parse_scan_config(&config).map_err(|error| privacy_error(py, error))? + } else { + core::ScanConfig::default() + }; + std::panic::catch_unwind(|| { + core::scan_with_config(text, &config) + .into_iter() + .map(Finding::from) + .collect() + }) + .map_err(|_| internal_error(py, "unexpected Rust scan failure")) } /// Transform explicit findings without scanning implicitly. @@ -353,26 +373,47 @@ fn transform( findings: Vec>, config: &Bound<'_, PyAny>, ) -> PyResult { - let strategy = parse_strategy(config)?; + let config = py_to_json(py, config, "")?; + let config = + core::parse_transformation_config(&config).map_err(|error| privacy_error(py, error))?; let core_findings: Vec = findings .iter() .map(|finding| finding.bind(py).borrow().to_core()) .collect(); - core::transform(text, &core_findings, strategy) + core::transform(text, &core_findings, &config) .map(TransformResult::from) - .map_err(|error| PyValueError::new_err(error.to_string())) + .map_err(|error| privacy_error(py, error)) } /// Scan text and transform the detected findings. #[pyfunction] -fn scan_and_transform(text: &str, config: &Bound<'_, PyAny>) -> PyResult { - core::scan_and_transform(text, parse_strategy(config)?) +fn scan_and_transform( + py: Python<'_>, + text: &str, + config: &Bound<'_, PyAny>, +) -> PyResult { + let config = py_to_json(py, config, "")?; + let config = + core::parse_scan_and_transform_config(&config).map_err(|error| privacy_error(py, error))?; + core::scan_and_transform(text, &config) .map(TransformResult::from) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) + .map_err(|error| privacy_error(py, error)) } #[pymodule] fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "DataFogConfigurationError", + module.py().get_type::(), + )?; + module.add( + "DataFogFindingError", + module.py().get_type::(), + )?; + module.add( + "DataFogInternalError", + module.py().get_type::(), + )?; module.add_class::()?; module.add_class::()?; module.add_class::()?; diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index bba00a4..c2b40fe 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -5,7 +5,15 @@ import json from pathlib import Path -from datafog_core import Finding, TextRange, scan, scan_and_transform, transform +from datafog_core import ( + DataFogConfigurationError, + DataFogFindingError, + Finding, + TextRange, + scan, + scan_and_transform, + transform, +) ROOT = Path(__file__).resolve().parents[3] @@ -59,10 +67,15 @@ def main() -> None: emoji_finding = scan("👋 jane@example.com")[0] assert (emoji_finding.byte_range.start, emoji_finding.byte_range.end) == (5, 21) assert (emoji_finding.codepoint_range.start, emoji_finding.codepoint_range.end) == (2, 18) + assert scan("Email jane@example.com", {"locale": "en-US"}) == scan( + "Email jane@example.com" + ) text = "👋 jane@example.com and jane@example.com" - explicit = transform(text, scan(text), {"strategy": "redact"}) - convenience = scan_and_transform(text, {"strategy": "redact"}) + explicit = transform(text, scan(text), {"default": {"strategy": "redact"}}) + convenience = scan_and_transform( + text, {"transform": {"default": {"strategy": "redact"}}} + ) assert explicit == convenience assert explicit.text == "👋 [EMAIL] and [EMAIL]" assert len(explicit.transformations) == 2 @@ -73,16 +86,20 @@ def main() -> None: masked = scan_and_transform( "Email jane@example.com", - {"strategy": "mask"}, + {"transform": {"default": {"strategy": "mask"}}}, ) assert masked.text == "Email ****************" partially_masked = scan_and_transform( "Email jane@example.com", { - "strategy": "mask", - "character": "•", - "reveal": {"direction": "last", "count": 4}, + "transform": { + "default": { + "strategy": "mask", + "character": "•", + "reveal": {"direction": "last", "count": 4}, + } + } }, ) assert partially_masked.text == "Email ••••••••••••.com" @@ -96,15 +113,19 @@ def main() -> None: unchanged = scan_and_transform( "Email jane@example.com", { - "strategy": "mask", - "reveal": {"direction": "first", "count": 99}, + "transform": { + "default": { + "strategy": "mask", + "reveal": {"direction": "first", "count": 99}, + } + } }, ) assert unchanged.text == "Email jane@example.com" removed = scan_and_transform( "Email jane@example.com today", - {"strategy": "remove"}, + {"transform": {"default": {"strategy": "remove"}}}, ) assert removed.text == "Email today" assert removed.transformations[0].strategy == "remove" @@ -135,9 +156,13 @@ def main() -> None: ] for invalid_config in invalid_configs: try: - scan_and_transform("Email jane@example.com", invalid_config) - except ValueError: - pass + scan_and_transform( + "Email jane@example.com", + {"transform": {"default": invalid_config}}, + ) + except DataFogConfigurationError as error: + assert error.code == "invalid_configuration" + assert error.path.startswith("/transform/default") else: raise AssertionError(f"invalid configuration was accepted: {invalid_config}") @@ -150,12 +175,59 @@ def main() -> None: confidence=2.0, ) try: - transform(text, [invalid], {"strategy": "redact"}) - except ValueError as error: - assert "InvalidConfidence" in str(error) + transform(text, [invalid], {"default": {"strategy": "redact"}}) + except DataFogFindingError as error: + assert error.code == "invalid_finding" + assert error.reason == "invalid_confidence" + assert error.path == "/findings/0/confidence" + assert error.finding_index == 0 else: raise AssertionError("invalid caller-supplied finding was accepted") + selected = scan_and_transform( + "Email support@example.com or call (212) 555-0100", + { + "scan": {"locale": "en-US"}, + "transform": { + "default": {"strategy": "redact"}, + "entities": ["EMAIL", "PHONE"], + "overrides": { + "PHONE": { + "strategy": "mask", + "reveal": {"direction": "last", "count": 4}, + } + }, + "allow": { + "exact": {"EMAIL": ["support@example.com"]}, + "regex": {}, + }, + }, + }, + ) + assert selected.text == "Email support@example.com or call **********0100" + assert len(selected.transformations) == 1 + + try: + transform( + text, + scan(text), + {"default": {"strategy": "redact"}, "overides": {}}, + ) + except DataFogConfigurationError as error: + assert error.reason == "unknown_field" + assert error.path == "/overides" + else: + raise AssertionError("unknown configuration field was accepted") + + try: + transform(text, scan(text), {"default": object()}) + except DataFogConfigurationError as error: + assert error.code == "invalid_configuration" + assert error.reason == "invalid_type" + assert error.path == "/default" + else: + raise AssertionError("non-JSON configuration value was accepted") + print("Installed datafog_core wheel matches fixtures and transform contracts.") diff --git a/bindings/wasm/Cargo.toml b/bindings/wasm/Cargo.toml index d591e4a..01952f2 100644 --- a/bindings/wasm/Cargo.toml +++ b/bindings/wasm/Cargo.toml @@ -14,4 +14,5 @@ crate-type = ["cdylib"] datafog-core = { path = "../../crates/core" } serde = { version = "1", features = ["derive"] } serde-wasm-bindgen = "0.6" +serde_json = "1" wasm-bindgen = "0.2" diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index 0f55b2b..e0a2410 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -7,7 +7,7 @@ export interface MaskRevealConfig { readonly count: number; } -export type TransformationConfig = +export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } | { @@ -16,6 +16,44 @@ export type TransformationConfig = readonly reveal?: MaskRevealConfig; }; +export interface RegexAllowRule { + readonly pattern: string; + readonly case_sensitive?: boolean; +} + +export interface AllowConfig { + readonly exact?: Readonly>; + readonly regex?: Readonly>; +} + +export interface TransformationConfig { + readonly default: TransformationStrategyConfig; + readonly entities?: readonly EntityType[]; + readonly overrides?: Readonly>; + readonly allow?: AllowConfig; +} + +export interface ScanConfig { + readonly locale?: string; +} + +export interface ScanAndTransformConfig { + readonly scan?: ScanConfig; + readonly transform: TransformationConfig; +} + +export type DataFogErrorCode = + | "invalid_configuration" + | "invalid_finding" + | "internal_error"; + +export declare class DataFogError extends Error { + readonly code: DataFogErrorCode; + readonly reason?: string; + readonly path?: string; + readonly findingIndex?: number; +} + export interface TextRange { readonly start: number; readonly end: number; @@ -45,7 +83,7 @@ export interface TransformResult { } export function init(): Promise; -export function scan(text: string): Finding[]; +export function scan(text: string, config?: ScanConfig): Finding[]; export function transform( text: string, findings: Finding[], @@ -53,5 +91,5 @@ export function transform( ): TransformResult; export function scanAndTransform( text: string, - config: TransformationConfig, + config: ScanAndTransformConfig, ): TransformResult; diff --git a/bindings/wasm/index.js b/bindings/wasm/index.js index bc1292f..37df521 100644 --- a/bindings/wasm/index.js +++ b/bindings/wasm/index.js @@ -24,7 +24,40 @@ export function init() { return initialization; } -export function scan(text) { +export class DataFogError extends Error { + constructor({ code, reason, message, path, findingIndex }) { + super(message); + this.name = "DataFogError"; + this.code = code; + this.reason = reason; + this.path = path; + this.findingIndex = findingIndex; + } +} + +function normalizeError(error, fallbackCode) { + if (error instanceof DataFogError) return error; + const source = error instanceof Error ? error.message : String(error); + try { + const details = JSON.parse(source); + if (typeof details.code === "string" && typeof details.message === "string") { + return new DataFogError(details); + } + } catch { + // Raw WASM conversion errors use the operation-specific fallback below. + } + return new DataFogError({ + code: fallbackCode, + reason: fallbackCode === "invalid_configuration" ? "invalid_type" : undefined, + message: + fallbackCode === "invalid_configuration" + ? "request configuration could not be decoded" + : "the WASM operation failed unexpectedly", + path: fallbackCode === "invalid_configuration" ? "" : undefined, + }); +} + +export function scan(text, config) { if (typeof text !== "string") { throw new TypeError("scan text must be a string"); } @@ -33,7 +66,14 @@ export function scan(text) { throw new Error("Call and await init() before scan()."); } - return scanWasm(text); + try { + return scanWasm(text, config); + } catch (error) { + throw normalizeError( + error, + config === undefined ? "internal_error" : "invalid_configuration", + ); + } } function assertInitialized(operation) { @@ -42,64 +82,6 @@ function assertInitialized(operation) { } } -function validateConfig(config) { - if (typeof config !== "object" || config === null || Array.isArray(config)) { - throw new TypeError("transformation configuration must be an object"); - } - if (!["redact", "mask", "remove"].includes(config.strategy)) { - throw new TypeError("strategy must be 'redact', 'mask', or 'remove'"); - } - - const allowed = - config.strategy === "mask" - ? new Set(["strategy", "character", "reveal"]) - : new Set(["strategy"]); - for (const key of Object.keys(config)) { - if (!allowed.has(key)) { - throw new TypeError(`unexpected configuration field: ${key}`); - } - } - - if (config.strategy === "mask") { - if (config.character !== undefined) { - if ( - typeof config.character !== "string" || - Array.from(config.character).length !== 1 || - /[\p{White_Space}\p{Cc}]/u.test(config.character) - ) { - throw new TypeError( - "mask character must be one non-whitespace, non-control code point", - ); - } - } - if (config.reveal !== undefined) { - if ( - typeof config.reveal !== "object" || - config.reveal === null || - Array.isArray(config.reveal) - ) { - throw new TypeError("mask reveal configuration must be an object"); - } - for (const key of Object.keys(config.reveal)) { - if (key !== "direction" && key !== "count") { - throw new TypeError(`unexpected reveal field: ${key}`); - } - } - if (!["first", "last"].includes(config.reveal.direction)) { - throw new TypeError("reveal direction must be 'first' or 'last'"); - } - if ( - !Number.isSafeInteger(config.reveal.count) || - config.reveal.count < 0 - ) { - throw new TypeError("reveal count must be a non-negative safe integer"); - } - } - } - - return config; -} - export function transform(text, findings, config) { if (typeof text !== "string") { throw new TypeError("transform text must be a string"); @@ -110,9 +92,9 @@ export function transform(text, findings, config) { assertInitialized("transform"); try { - return transformWasm(text, findings, validateConfig(config)); + return transformWasm(text, findings, config); } catch (error) { - throw error instanceof Error ? error : new Error(String(error)); + throw normalizeError(error, "invalid_configuration"); } } @@ -123,8 +105,8 @@ export function scanAndTransform(text, config) { assertInitialized("scanAndTransform"); try { - return scanAndTransformWasm(text, validateConfig(config)); + return scanAndTransformWasm(text, config); } catch (error) { - throw error instanceof Error ? error : new Error(String(error)); + throw normalizeError(error, "invalid_configuration"); } } diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 9b05e55..e62217f 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -64,31 +64,6 @@ struct TransformResult { transformations: Vec, } -#[derive(Deserialize)] -#[serde(rename_all = "lowercase")] -enum RevealDirection { - First, - Last, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct MaskRevealConfig { - direction: RevealDirection, - count: usize, -} - -#[derive(Deserialize)] -#[serde(tag = "strategy", rename_all = "lowercase", deny_unknown_fields)] -enum TransformationConfig { - Redact, - Remove, - Mask { - character: Option, - reveal: Option, - }, -} - fn finding_from_core(finding: datafog_core::Finding) -> Finding { Finding { entity_type: finding.entity_type, @@ -101,41 +76,21 @@ fn finding_from_core(finding: datafog_core::Finding) -> Finding { } } -fn strategy_from_js(config: JsValue) -> Result { - let config: TransformationConfig = serde_wasm_bindgen::from_value(config) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - match config { - TransformationConfig::Redact => Ok(datafog_core::TransformationStrategy::Redact), - TransformationConfig::Remove => Ok(datafog_core::TransformationStrategy::Remove), - TransformationConfig::Mask { character, reveal } => { - let character = character.unwrap_or_else(|| "*".to_owned()); - let mut characters = character.chars(); - let character = characters - .next() - .filter(|_| characters.next().is_none()) - .ok_or_else(|| { - JsValue::from_str("mask character must contain exactly one code point") - })?; - let reveal = match reveal { - None => datafog_core::MaskReveal::None, - Some(MaskRevealConfig { - direction: RevealDirection::First, - count, - }) => datafog_core::MaskReveal::First(count), - Some(MaskRevealConfig { - direction: RevealDirection::Last, - count, - }) => datafog_core::MaskReveal::Last(count), - }; - datafog_core::MaskConfig::new(character, reveal) - .map(datafog_core::TransformationStrategy::Mask) - .map_err(|_| { - JsValue::from_str( - "mask character must not be whitespace or a control character", - ) - }) - } - } +fn privacy_error(error: datafog_core::PrivacyError) -> JsValue { + JsValue::from_str( + &serde_json::json!({ + "code": error.code().as_str(), + "reason": error.reason().map(datafog_core::PrivacyErrorReason::as_str), + "message": error.to_string(), + "path": error.path(), + "findingIndex": error.finding_index(), + }) + .to_string(), + ) +} + +fn config_value(config: JsValue) -> Result { + serde_wasm_bindgen::from_value(config).map_err(|error| JsValue::from_str(&error.to_string())) } fn result_to_js(result: datafog_core::TransformResult) -> Result { @@ -162,8 +117,14 @@ fn result_to_js(result: datafog_core::TransformResult) -> Result Result { - let findings: Vec = datafog_core::scan(text) +pub fn scan(text: &str, config: Option) -> Result { + let config = if let Some(config) = config { + let config = config_value(config)?; + datafog_core::parse_scan_config(&config).map_err(privacy_error)? + } else { + datafog_core::ScanConfig::default() + }; + let findings: Vec = datafog_core::scan_with_config(text, &config) .into_iter() .map(finding_from_core) .collect(); @@ -179,14 +140,16 @@ pub fn transform(text: &str, findings: JsValue, config: JsValue) -> Result>(); - let result = datafog_core::transform(text, &findings, strategy_from_js(config)?) - .map_err(|error| JsValue::from_str(&error.to_string()))?; + let config = config_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(privacy_error)?; + let result = datafog_core::transform(text, &findings, &config).map_err(privacy_error)?; result_to_js(result) } #[wasm_bindgen] pub fn scan_and_transform(text: &str, config: JsValue) -> Result { - let result = datafog_core::scan_and_transform(text, strategy_from_js(config)?) - .map_err(|error| JsValue::from_str(&error.to_string()))?; + let config = config_value(config)?; + let config = datafog_core::parse_scan_and_transform_config(&config).map_err(privacy_error)?; + let result = datafog_core::scan_and_transform(text, &config).map_err(privacy_error)?; result_to_js(result) } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index cdec031..fd66dae 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,5 +1,6 @@ //! Core PII scanning API for DataFog. -use regex::Regex; +use regex::{Regex, RegexSet, RegexSetBuilder}; +use std::collections::{BTreeMap, BTreeSet}; use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::LazyLock; @@ -43,6 +44,672 @@ pub enum TransformationStrategy { Mask(MaskConfig), } +const MAX_REGEX_RULES: usize = 100; +const MAX_REGEX_PATTERN_BYTES: usize = 1024; +const MAX_REGEX_SOURCE_BYTES: usize = 10 * 1024; +const MAX_COMPILED_REGEX_GROUP_BYTES: usize = 1024 * 1024; + +/// One entity-scoped full-match regex allowlist rule. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RegexAllowRule { + pattern: String, + case_sensitive: bool, +} + +impl RegexAllowRule { + /// Create a rule. Complete validation occurs when the rule is added to a + /// transformation configuration. + pub fn new(pattern: impl Into, case_sensitive: bool) -> Self { + Self { + pattern: pattern.into(), + case_sensitive, + } + } + + /// Regex source supplied by the caller. + pub fn pattern(&self) -> &str { + &self.pattern + } + + /// Whether matching preserves case distinctions. + pub fn case_sensitive(&self) -> bool { + self.case_sensitive + } +} + +#[derive(Debug, Clone)] +struct CompiledRegexGroup { + patterns: RegexSet, +} + +/// Validated configuration for transforming caller-supplied findings. +#[derive(Debug, Clone)] +pub struct TransformationConfig { + default: TransformationStrategy, + entities: Option>, + overrides: BTreeMap, + exact_allowlists: BTreeMap>, + regex_allowlists: BTreeMap>, + compiled_regex_allowlists: BTreeMap>, +} + +impl TransformationConfig { + /// Create a configuration which applies one default strategy to all + /// supplied findings. + pub fn new(default: TransformationStrategy) -> Self { + Self { + default, + entities: None, + overrides: BTreeMap::new(), + exact_allowlists: BTreeMap::new(), + regex_allowlists: BTreeMap::new(), + compiled_regex_allowlists: BTreeMap::new(), + } + } + + /// Restrict transformation to a non-empty set of exact entity types. + pub fn with_entities(mut self, entities: Vec) -> Result { + if entities.is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + "/entities", + "entities must contain at least one entity type", + )); + } + let mut selected = BTreeSet::new(); + for (index, entity) in entities.into_iter().enumerate() { + validate_entity_name(&entity, &format!("/entities/{index}"))?; + if !selected.insert(entity) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::DuplicateValue, + format!("/entities/{index}"), + "entity selection contains a duplicate entity type", + )); + } + } + self.entities = Some(selected); + Ok(self) + } + + /// Add an exact, case-sensitive entity strategy override. + pub fn with_override( + mut self, + entity_type: impl Into, + strategy: TransformationStrategy, + ) -> Result { + let entity_type = entity_type.into(); + let path = format!("/overrides/{}", json_pointer_segment(&entity_type)); + validate_entity_name(&entity_type, &path)?; + if self.overrides.insert(entity_type, strategy).is_some() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::DuplicateValue, + path, + "entity type has more than one strategy override", + )); + } + Ok(self) + } + + /// Add exact, case-sensitive allowlist values for one entity type. + pub fn with_exact_allowlist( + mut self, + entity_type: impl Into, + values: Vec, + ) -> Result { + let entity_type = entity_type.into(); + let path = format!("/allow/exact/{}", json_pointer_segment(&entity_type)); + validate_entity_name(&entity_type, &path)?; + if values.is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + path, + "exact allowlist must contain at least one value", + )); + } + if self.exact_allowlists.contains_key(&entity_type) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::DuplicateValue, + path, + "entity type has more than one exact allowlist", + )); + } + let mut deduplicated = BTreeSet::new(); + for (index, value) in values.into_iter().enumerate() { + if value.is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + format!("{path}/{index}"), + "exact allowlist values must not be empty", + )); + } + deduplicated.insert(value); + } + self.exact_allowlists.insert(entity_type, deduplicated); + Ok(self) + } + + /// Add full-match regex allowlist rules for one entity type. + pub fn with_regex_allowlist( + mut self, + entity_type: impl Into, + rules: Vec, + ) -> Result { + let entity_type = entity_type.into(); + let path = format!("/allow/regex/{}", json_pointer_segment(&entity_type)); + validate_entity_name(&entity_type, &path)?; + if rules.is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + path, + "regex allowlist must contain at least one rule", + )); + } + if self.regex_allowlists.contains_key(&entity_type) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::DuplicateValue, + path, + "entity type has more than one regex allowlist", + )); + } + let rules = rules + .into_iter() + .collect::>() + .into_iter() + .collect(); + self.regex_allowlists.insert(entity_type, rules); + self.compile_regex_allowlists()?; + Ok(self) + } + + fn includes(&self, finding: &Finding) -> bool { + self.entities + .as_ref() + .is_none_or(|entities| entities.contains(&finding.entity_type)) + } + + fn allows(&self, finding: &Finding) -> bool { + self.exact_allowlists + .get(&finding.entity_type) + .is_some_and(|values| values.contains(&finding.matched_text)) + || self + .compiled_regex_allowlists + .get(&finding.entity_type) + .is_some_and(|groups| { + groups + .iter() + .any(|group| group.patterns.is_match(&finding.matched_text)) + }) + } + + fn strategy_for(&self, finding: &Finding) -> TransformationStrategy { + self.overrides + .get(&finding.entity_type) + .copied() + .unwrap_or(self.default) + } + + fn compile_regex_allowlists(&mut self) -> Result<(), PrivacyError> { + let rule_count: usize = self.regex_allowlists.values().map(Vec::len).sum(); + if rule_count > MAX_REGEX_RULES { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::LimitExceeded, + "/allow/regex", + "regex allowlists exceed the maximum of 100 deduplicated rules", + )); + } + + let mut source_bytes = 0; + for (entity_type, rules) in &self.regex_allowlists { + let entity_path = format!("/allow/regex/{}", json_pointer_segment(entity_type)); + for (index, rule) in rules.iter().enumerate() { + let pattern_path = format!("{entity_path}/{index}/pattern"); + if rule.pattern.is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + pattern_path, + "regex pattern must not be empty", + )); + } + if rule.pattern.len() > MAX_REGEX_PATTERN_BYTES { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::LimitExceeded, + pattern_path, + "regex pattern exceeds the 1 KiB source limit", + )); + } + source_bytes += rule.pattern.len(); + } + } + if source_bytes > MAX_REGEX_SOURCE_BYTES { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::LimitExceeded, + "/allow/regex", + "regex allowlists exceed the 10 KiB aggregate source limit", + )); + } + + let mut compiled = BTreeMap::new(); + for (entity_type, rules) in &self.regex_allowlists { + let entity_path = format!("/allow/regex/{}", json_pointer_segment(entity_type)); + let mut by_case = BTreeMap::>::new(); + for rule in rules { + by_case + .entry(rule.case_sensitive) + .or_default() + .push(format!(r"\A(?:{})\z", rule.pattern)); + } + + let mut groups = Vec::new(); + for (case_sensitive, patterns) in by_case { + let set = RegexSetBuilder::new(patterns) + .case_insensitive(!case_sensitive) + .size_limit(MAX_COMPILED_REGEX_GROUP_BYTES) + .dfa_size_limit(MAX_COMPILED_REGEX_GROUP_BYTES) + .build() + .map_err(|error| { + let reason = match error { + regex::Error::CompiledTooBig(_) => PrivacyErrorReason::LimitExceeded, + _ => PrivacyErrorReason::InvalidRegex, + }; + PrivacyError::invalid_configuration( + reason, + &entity_path, + "regex allowlist contains an invalid or over-limit pattern", + ) + })?; + groups.push(CompiledRegexGroup { patterns: set }); + } + compiled.insert(entity_type.clone(), groups); + } + self.compiled_regex_allowlists = compiled; + Ok(()) + } +} + +fn validate_entity_name(entity_type: &str, path: &str) -> Result<(), PrivacyError> { + if entity_type.trim().is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + path, + "entity type must not be empty or whitespace-only", + )); + } + Ok(()) +} + +fn json_pointer_segment(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") +} + +/// Scanner configuration. Current built-in detectors share the same execution +/// path; locale is retained for detector-specific routing as coverage expands. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ScanConfig { + locale: Option, +} + +impl ScanConfig { + /// Create scanner configuration using detector defaults. + pub fn new() -> Self { + Self::default() + } + + /// Set a non-empty locale identifier. + pub fn with_locale(mut self, locale: impl Into) -> Result { + let locale = locale.into(); + if locale.trim().is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + "/locale", + "scan locale must not be empty or whitespace-only", + )); + } + self.locale = Some(locale); + Ok(self) + } + + /// Configured locale, when explicitly supplied. + pub fn locale(&self) -> Option<&str> { + self.locale.as_deref() + } +} + +/// Configuration for the explicit scan-then-transform convenience operation. +#[derive(Debug, Clone)] +pub struct ScanAndTransformConfig { + scan: ScanConfig, + transform: TransformationConfig, +} + +impl ScanAndTransformConfig { + /// Use scanner defaults with a required transformation configuration. + pub fn new(transform: TransformationConfig) -> Self { + Self { + scan: ScanConfig::default(), + transform, + } + } + + /// Supply scanner configuration. + pub fn with_scan(mut self, scan: ScanConfig) -> Self { + self.scan = scan; + self + } + + /// Scanner settings. + pub fn scan_config(&self) -> &ScanConfig { + &self.scan + } + + /// Transformation settings. + pub fn transformation_config(&self) -> &TransformationConfig { + &self.transform + } +} + +/// Parse the canonical serialized transformation envelope. +pub fn parse_transformation_config( + value: &serde_json::Value, +) -> Result { + let object = require_object(value, "", "transformation configuration must be an object")?; + reject_unknown_fields(object, &["default", "entities", "overrides", "allow"], "")?; + let default = object.get("default").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + "/default", + "transformation configuration requires default", + ) + })?; + let mut config = TransformationConfig::new(parse_strategy_config(default, "/default")?); + + if let Some(entities) = object.get("entities") { + let entities = require_array(entities, "/entities", "entities must be an array")?; + let mut parsed = Vec::with_capacity(entities.len()); + for (index, entity) in entities.iter().enumerate() { + parsed.push(require_string( + entity, + &format!("/entities/{index}"), + "entity type must be a string", + )?); + } + config = config.with_entities(parsed)?; + } + + if let Some(overrides) = object.get("overrides") { + let overrides = require_object( + overrides, + "/overrides", + "strategy overrides must be an object", + )?; + for (entity_type, strategy) in overrides { + let path = format!("/overrides/{}", json_pointer_segment(entity_type)); + config = config.with_override(entity_type, parse_strategy_config(strategy, &path)?)?; + } + } + + if let Some(allow) = object.get("allow") { + let allow = require_object(allow, "/allow", "allow must be an object")?; + reject_unknown_fields(allow, &["exact", "regex"], "/allow")?; + if let Some(exact) = allow.get("exact") { + let exact = + require_object(exact, "/allow/exact", "exact allowlists must be an object")?; + for (entity_type, values) in exact { + let entity_path = format!("/allow/exact/{}", json_pointer_segment(entity_type)); + let values = + require_array(values, &entity_path, "exact allowlist must be an array")?; + let mut parsed = Vec::with_capacity(values.len()); + for (index, value) in values.iter().enumerate() { + parsed.push(require_string( + value, + &format!("{entity_path}/{index}"), + "exact allowlist value must be a string", + )?); + } + config = config.with_exact_allowlist(entity_type, parsed)?; + } + } + if let Some(regex) = allow.get("regex") { + let regex = + require_object(regex, "/allow/regex", "regex allowlists must be an object")?; + for (entity_type, rules) in regex { + let entity_path = format!("/allow/regex/{}", json_pointer_segment(entity_type)); + let rules = require_array(rules, &entity_path, "regex allowlist must be an array")?; + let mut parsed = Vec::with_capacity(rules.len()); + for (index, rule) in rules.iter().enumerate() { + let path = format!("{entity_path}/{index}"); + let rule = require_object(rule, &path, "regex rule must be an object")?; + reject_unknown_fields(rule, &["pattern", "case_sensitive"], &path)?; + let pattern = rule.get("pattern").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + format!("{path}/pattern"), + "regex rule requires pattern", + ) + })?; + let pattern = require_string( + pattern, + &format!("{path}/pattern"), + "regex pattern must be a string", + )?; + let case_sensitive = match rule.get("case_sensitive") { + None => true, + Some(serde_json::Value::Bool(value)) => *value, + Some(_) => { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidType, + format!("{path}/case_sensitive"), + "case_sensitive must be a boolean", + )); + } + }; + parsed.push(RegexAllowRule::new(pattern, case_sensitive)); + } + config = config.with_regex_allowlist(entity_type, parsed)?; + } + } + } + Ok(config) +} + +/// Parse the canonical divided scan-and-transform envelope. +pub fn parse_scan_and_transform_config( + value: &serde_json::Value, +) -> Result { + let object = require_object( + value, + "", + "scan-and-transform configuration must be an object", + )?; + reject_unknown_fields(object, &["scan", "transform"], "")?; + let transform = object.get("transform").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + "/transform", + "scan-and-transform configuration requires transform", + ) + })?; + let transform = + parse_transformation_config(transform).map_err(|error| error.prefixed("/transform"))?; + let mut combined = ScanAndTransformConfig::new(transform); + if let Some(scan) = object.get("scan") { + let scan_config = parse_scan_config(scan).map_err(|error| error.prefixed("/scan"))?; + combined = combined.with_scan(scan_config); + } + Ok(combined) +} + +/// Parse canonical scanner configuration. +pub fn parse_scan_config(value: &serde_json::Value) -> Result { + let object = require_object(value, "", "scan configuration must be an object")?; + reject_unknown_fields(object, &["locale"], "")?; + let mut config = ScanConfig::new(); + if let Some(locale) = object.get("locale") { + let locale = require_string(locale, "/locale", "scan locale must be a string")?; + config = config.with_locale(locale)?; + } + Ok(config) +} + +fn parse_strategy_config( + value: &serde_json::Value, + path: &str, +) -> Result { + let object = require_object(value, path, "strategy configuration must be an object")?; + let strategy_path = format!("{path}/strategy"); + let strategy = object.get("strategy").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + &strategy_path, + "strategy configuration requires strategy", + ) + })?; + let strategy = require_string(strategy, &strategy_path, "strategy must be a string")?; + match strategy.as_str() { + "redact" => { + reject_unknown_fields(object, &["strategy"], path)?; + Ok(TransformationStrategy::Redact) + } + "remove" => { + reject_unknown_fields(object, &["strategy"], path)?; + Ok(TransformationStrategy::Remove) + } + "mask" => { + reject_unknown_fields(object, &["strategy", "character", "reveal"], path)?; + let character = match object.get("character") { + None => '*', + Some(value) => { + let value = require_string( + value, + &format!("{path}/character"), + "mask character must be a string", + )?; + let mut characters = value.chars(); + characters + .next() + .filter(|_| characters.next().is_none()) + .ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + format!("{path}/character"), + "mask character must contain exactly one code point", + ) + })? + } + }; + let reveal = match object.get("reveal") { + None => MaskReveal::None, + Some(value) => { + let reveal_path = format!("{path}/reveal"); + let reveal = + require_object(value, &reveal_path, "mask reveal must be an object")?; + reject_unknown_fields(reveal, &["direction", "count"], &reveal_path)?; + let direction = reveal.get("direction").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + format!("{reveal_path}/direction"), + "mask reveal requires direction", + ) + })?; + let direction = require_string( + direction, + &format!("{reveal_path}/direction"), + "mask reveal direction must be a string", + )?; + let count = reveal.get("count").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + format!("{reveal_path}/count"), + "mask reveal requires count", + ) + })?; + let count = count + .as_u64() + .and_then(|count| usize::try_from(count).ok()) + .ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + format!("{reveal_path}/count"), + "mask reveal count must be a non-negative integer", + ) + })?; + match direction.as_str() { + "first" => MaskReveal::First(count), + "last" => MaskReveal::Last(count), + _ => { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + format!("{reveal_path}/direction"), + "mask reveal direction must be first or last", + )); + } + } + } + }; + MaskConfig::new(character, reveal) + .map(TransformationStrategy::Mask) + .map_err(|_| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + format!("{path}/character"), + "mask character must not be whitespace or a control character", + ) + }) + } + _ => Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + strategy_path, + "strategy must be redact, mask, or remove", + )), + } +} + +fn require_object<'a>( + value: &'a serde_json::Value, + path: &str, + message: &str, +) -> Result<&'a serde_json::Map, PrivacyError> { + value.as_object().ok_or_else(|| { + PrivacyError::invalid_configuration(PrivacyErrorReason::InvalidType, path, message) + }) +} + +fn require_array<'a>( + value: &'a serde_json::Value, + path: &str, + message: &str, +) -> Result<&'a [serde_json::Value], PrivacyError> { + value.as_array().map(Vec::as_slice).ok_or_else(|| { + PrivacyError::invalid_configuration(PrivacyErrorReason::InvalidType, path, message) + }) +} + +fn require_string( + value: &serde_json::Value, + path: &str, + message: &str, +) -> Result { + value.as_str().map(str::to_owned).ok_or_else(|| { + PrivacyError::invalid_configuration(PrivacyErrorReason::InvalidType, path, message) + }) +} + +fn reject_unknown_fields( + object: &serde_json::Map, + allowed: &[&str], + path: &str, +) -> Result<(), PrivacyError> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::UnknownField, + format!("{path}/{}", json_pointer_segment(key)), + "configuration contains an unknown field", + )); + } + } + Ok(()) +} + /// Configuration for character masking. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MaskConfig { @@ -154,26 +821,179 @@ pub enum FindingValidationError { InvalidConfidence, } -/// A transformation request could not be completed. +/// Stable top-level category for a privacy-operation error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrivacyErrorCode { + /// Transformation or scanning configuration is invalid. + InvalidConfiguration, + /// One caller-supplied finding is invalid. + InvalidFinding, + /// An unexpected non-caller-correctable failure occurred. + InternalError, +} + +impl PrivacyErrorCode { + /// Stable serialized value. + pub fn as_str(self) -> &'static str { + match self { + Self::InvalidConfiguration => "invalid_configuration", + Self::InvalidFinding => "invalid_finding", + Self::InternalError => "internal_error", + } + } +} + +/// Stable machine-readable reason for a caller-correctable error. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct TransformError { - /// Index of the invalid finding in the caller-supplied slice. - pub finding_index: usize, - /// Validation failure. - pub kind: FindingValidationError, +pub enum PrivacyErrorReason { + MissingField, + UnknownField, + InvalidType, + InvalidValue, + EmptyValue, + DuplicateValue, + InvalidRegex, + LimitExceeded, + MatchedTextMismatch, + InconsistentRanges, + OutOfBounds, + InvalidBoundary, + InvalidConfidence, +} + +impl PrivacyErrorReason { + /// Stable serialized value. + pub fn as_str(self) -> &'static str { + match self { + Self::MissingField => "missing_field", + Self::UnknownField => "unknown_field", + Self::InvalidType => "invalid_type", + Self::InvalidValue => "invalid_value", + Self::EmptyValue => "empty_value", + Self::DuplicateValue => "duplicate_value", + Self::InvalidRegex => "invalid_regex", + Self::LimitExceeded => "limit_exceeded", + Self::MatchedTextMismatch => "matched_text_mismatch", + Self::InconsistentRanges => "inconsistent_ranges", + Self::OutOfBounds => "out_of_bounds", + Self::InvalidBoundary => "invalid_boundary", + Self::InvalidConfidence => "invalid_confidence", + } + } +} + +/// A privacy operation could not be completed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrivacyError { + code: PrivacyErrorCode, + reason: Option, + path: Option, + finding_index: Option, + message: String, } -impl std::fmt::Display for TransformError { +impl PrivacyError { + fn invalid_configuration( + reason: PrivacyErrorReason, + path: impl Into, + message: impl Into, + ) -> Self { + Self { + code: PrivacyErrorCode::InvalidConfiguration, + reason: Some(reason), + path: Some(path.into()), + finding_index: None, + message: message.into(), + } + } + + fn invalid_finding(finding_index: usize, kind: FindingValidationError) -> Self { + let (reason, suffix, message) = match kind { + FindingValidationError::EmptyOrReversedByteRange => ( + PrivacyErrorReason::InvalidValue, + "byte_range", + "finding byte range must be non-empty and increasing", + ), + FindingValidationError::ByteRangeOutOfBounds => ( + PrivacyErrorReason::OutOfBounds, + "byte_range", + "finding byte range is outside the source text", + ), + FindingValidationError::InvalidUtf8Boundary => ( + PrivacyErrorReason::InvalidBoundary, + "byte_range", + "finding byte range does not use UTF-8 boundaries", + ), + FindingValidationError::EmptyOrReversedCodepointRange => ( + PrivacyErrorReason::InvalidValue, + "codepoint_range", + "finding code-point range must be non-empty and increasing", + ), + FindingValidationError::CodepointRangeOutOfBounds => ( + PrivacyErrorReason::OutOfBounds, + "codepoint_range", + "finding code-point range is outside the source text", + ), + FindingValidationError::InconsistentRanges => ( + PrivacyErrorReason::InconsistentRanges, + "codepoint_range", + "finding byte and code-point ranges select different text", + ), + FindingValidationError::MatchedTextMismatch => ( + PrivacyErrorReason::MatchedTextMismatch, + "matched_text", + "finding text does not match the selected source span", + ), + FindingValidationError::InvalidConfidence => ( + PrivacyErrorReason::InvalidConfidence, + "confidence", + "finding confidence must be finite and in 0.0..=1.0", + ), + }; + Self { + code: PrivacyErrorCode::InvalidFinding, + reason: Some(reason), + path: Some(format!("/findings/{finding_index}/{suffix}")), + finding_index: Some(finding_index), + message: message.to_owned(), + } + } + + fn prefixed(mut self, prefix: &str) -> Self { + if let Some(path) = &mut self.path { + *path = format!("{prefix}{path}"); + } + self + } + + /// Stable top-level category. + pub fn code(&self) -> PrivacyErrorCode { + self.code + } + + /// Stable caller-correctable reason, when applicable. + pub fn reason(&self) -> Option { + self.reason + } + + /// RFC 6901 request path, when applicable. + pub fn path(&self) -> Option<&str> { + self.path.as_deref() + } + + /// Invalid finding index, when applicable. + pub fn finding_index(&self) -> Option { + self.finding_index + } +} + +impl std::fmt::Display for PrivacyError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "invalid finding at index {}: {:?}", - self.finding_index, self.kind - ) + formatter.write_str(&self.message) } } -impl std::error::Error for TransformError {} +impl std::error::Error for PrivacyError {} #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] enum Label { @@ -222,6 +1042,11 @@ struct Candidate { /// Scan text for supported PII findings. pub fn scan(text: &str) -> Vec { + scan_with_config(text, &ScanConfig::default()) +} + +/// Scan text using explicit detector configuration. +pub fn scan_with_config(text: &str, _config: &ScanConfig) -> Vec { let mut candidates: Vec = Vec::new(); detect_email(text, &mut candidates); detect_phone(text, &mut candidates); @@ -237,19 +1062,19 @@ pub fn scan(text: &str) -> Vec { pub fn transform( text: &str, findings: &[Finding], - strategy: TransformationStrategy, -) -> Result { + config: &TransformationConfig, +) -> Result { for (finding_index, finding) in findings.iter().enumerate() { if let Err(kind) = validate_finding(text, finding) { - return Err(TransformError { - finding_index, - kind, - }); + return Err(PrivacyError::invalid_finding(finding_index, kind)); } } let mut selected_findings: Vec = Vec::with_capacity(findings.len()); - for finding in findings { + for finding in findings + .iter() + .filter(|finding| config.includes(finding) && !config.allows(finding)) + { if let Some(existing) = selected_findings .iter_mut() .find(|existing| findings_are_duplicates(existing, finding)) @@ -278,6 +1103,7 @@ pub fn transform( output.push_str(&text[source_byte_cursor..finding.byte_range.start]); let output_byte_start = output.len(); let output_codepoint_start = output.chars().count(); + let strategy = config.strategy_for(finding); let replacement = match strategy { TransformationStrategy::Redact => format!("[{}]", finding.entity_type), TransformationStrategy::Remove => String::new(), @@ -328,9 +1154,13 @@ pub fn transform( /// Scan text and transform the resulting findings in one explicit convenience operation. pub fn scan_and_transform( text: &str, - strategy: TransformationStrategy, -) -> Result { - transform(text, &scan(text), strategy) + config: &ScanAndTransformConfig, +) -> Result { + transform( + text, + &scan_with_config(text, config.scan_config()), + config.transformation_config(), + ) } fn findings_are_duplicates(left: &Finding, right: &Finding) -> bool { @@ -1111,9 +1941,17 @@ fn detect_ip_address(text: &str, candidates: &mut Vec) { #[cfg(test)] mod tests { use super::{ - Finding, FindingValidationError, MaskConfig, MaskConfigError, MaskReveal, TextRange, - TransformError, TransformationStrategy, scan, scan_and_transform, transform, + Finding, FindingValidationError, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES, MaskConfig, + MaskConfigError, MaskReveal, PrivacyError, PrivacyErrorCode, PrivacyErrorReason, + RegexAllowRule, ScanAndTransformConfig, TextRange, TransformationConfig, + TransformationStrategy, parse_scan_and_transform_config, parse_transformation_config, scan, + scan_and_transform, transform, }; + use serde_json::json; + + fn config(strategy: TransformationStrategy) -> TransformationConfig { + TransformationConfig::new(strategy) + } fn expected_finding( entity_type: &str, @@ -1394,7 +2232,7 @@ mod tests { let text = "Contact jane@example.com"; let findings = scan(text); - let result = transform(text, &findings, TransformationStrategy::Redact).unwrap(); + let result = transform(text, &findings, &config(TransformationStrategy::Redact)).unwrap(); assert_eq!(result.text, "Contact [EMAIL]"); assert_eq!(result.transformations.len(), 1); @@ -1412,13 +2250,327 @@ mod tests { ); } + #[test] + fn entity_override_replaces_the_default_strategy() { + let text = "Email jane@example.com or call (212) 555-0100"; + let findings = scan(text); + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_override( + "PHONE", + TransformationStrategy::Mask(MaskConfig::new('*', MaskReveal::Last(4)).unwrap()), + ) + .unwrap(); + + let result = transform(text, &findings, &config).unwrap(); + + assert_eq!(result.text, "Email [EMAIL] or call **********0100"); + assert_eq!( + result.transformations[0].strategy, + TransformationStrategy::Redact + ); + assert!(matches!( + result.transformations[1].strategy, + TransformationStrategy::Mask(_) + )); + } + + #[test] + fn entity_selection_transforms_only_exact_selected_types() { + let text = "Email jane@example.com or call (212) 555-0100"; + let findings = scan(text); + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_entities(vec!["PHONE".to_owned()]) + .unwrap(); + + let result = transform(text, &findings, &config).unwrap(); + + assert_eq!(result.text, "Email jane@example.com or call [PHONE]"); + assert_eq!(result.transformations.len(), 1); + assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + } + + #[test] + fn exact_allowlist_is_entity_scoped_and_applied_before_overlap_resolution() { + let text = "212-555-0100@example.com"; + let email = supplied_ascii_finding(text, "EMAIL", 0, text.len(), Some(0.9), "email"); + let phone = supplied_ascii_finding(text, "PHONE", 0, 12, Some(0.8), "phone"); + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_exact_allowlist("EMAIL", vec![text.to_owned(), text.to_owned()]) + .unwrap(); + + let result = transform(text, &[email, phone], &config).unwrap(); + + assert_eq!(result.text, "[PHONE]@example.com"); + assert_eq!(result.transformations.len(), 1); + assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + } + + #[test] + fn entity_selection_happens_before_overlap_resolution() { + let text = "Acme Corporation"; + let unselected_outer = supplied_ascii_finding( + text, + "ORGANIZATION", + 0, + text.len(), + Some(0.9), + "organization", + ); + let selected_inner = supplied_ascii_finding(text, "PERSON", 0, 4, Some(0.8), "person"); + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_entities(vec!["PERSON".to_owned()]) + .unwrap(); + + let result = transform(text, &[unselected_outer, selected_inner], &config).unwrap(); + + assert_eq!(result.text, "[PERSON] Corporation"); + } + + #[test] + fn regex_allowlists_use_full_match_and_explicit_case_sensitivity() { + let text = "allowed@example.com ADMIN@EXAMPLE.COM"; + let lower = supplied_ascii_finding(text, "EMAIL", 0, 19, None, "test"); + let upper = supplied_ascii_finding(text, "EMAIL", 20, text.len(), None, "test"); + let sensitive = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("EMAIL", vec![RegexAllowRule::new(r".*@example\.com", true)]) + .unwrap(); + + let result = transform(text, &[lower.clone(), upper.clone()], &sensitive).unwrap(); + assert_eq!(result.text, "allowed@example.com [EMAIL]"); + + let insensitive = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist( + "EMAIL", + vec![ + RegexAllowRule::new(r".*@example\.com", false), + RegexAllowRule::new(r".*@example\.com", false), + ], + ) + .unwrap(); + let explicit = transform(text, &[lower, upper], &insensitive).unwrap(); + let convenience = + scan_and_transform(text, &ScanAndTransformConfig::new(insensitive)).unwrap(); + assert_eq!(explicit.text, text); + assert_eq!(convenience, explicit); + } + + #[test] + fn configuration_errors_expose_stable_machine_readable_fields() { + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_entities(Vec::new()) + .unwrap_err(); + + assert_eq!(error.code(), PrivacyErrorCode::InvalidConfiguration); + assert_eq!(error.reason(), Some(PrivacyErrorReason::EmptyValue)); + assert_eq!(error.path(), Some("/entities")); + assert_eq!(error.finding_index(), None); + } + + #[test] + fn canonical_serialized_envelope_drives_selection_overrides_and_allowlists() { + let text = "Email support@example.com or call (212) 555-0100"; + let findings = scan(text); + let config = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "entities": ["EMAIL", "PHONE"], + "overrides": { + "PHONE": { + "strategy": "mask", + "reveal": { "direction": "last", "count": 4 } + } + }, + "allow": { + "exact": { "EMAIL": ["support@example.com"] }, + "regex": {} + } + })) + .unwrap(); + + let result = transform(text, &findings, &config).unwrap(); + let convenience = scan_and_transform(text, &ScanAndTransformConfig::new(config)).unwrap(); + + assert_eq!( + result.text, + "Email support@example.com or call **********0100" + ); + assert_eq!(result.transformations.len(), 1); + assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + assert_eq!(convenience, result); + } + + #[test] + fn serialized_configuration_rejects_unknown_fields_and_null() { + let unknown = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "overides": {} + })) + .unwrap_err(); + assert_eq!(unknown.reason(), Some(PrivacyErrorReason::UnknownField)); + assert_eq!(unknown.path(), Some("/overides")); + + let explicit_null = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "allow": null + })) + .unwrap_err(); + assert_eq!( + explicit_null.reason(), + Some(PrivacyErrorReason::InvalidType) + ); + assert_eq!(explicit_null.path(), Some("/allow")); + } + + #[test] + fn serialized_configuration_distinguishes_empty_structure_from_empty_semantics() { + let accepted = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "overrides": {}, + "allow": { "exact": {}, "regex": {} } + })); + assert!(accepted.is_ok()); + + let duplicate_entity = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "entities": ["EMAIL", "EMAIL"] + })) + .unwrap_err(); + assert_eq!( + duplicate_entity.reason(), + Some(PrivacyErrorReason::DuplicateValue) + ); + assert_eq!(duplicate_entity.path(), Some("/entities/1")); + + let empty_allowlist = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "allow": { "exact": { "EMAIL": [] } } + })) + .unwrap_err(); + assert_eq!( + empty_allowlist.reason(), + Some(PrivacyErrorReason::EmptyValue) + ); + assert_eq!(empty_allowlist.path(), Some("/allow/exact/EMAIL")); + } + + #[test] + fn exact_allowlists_compare_unicode_values_without_normalizing_them() { + let text = "Name José"; + let finding = Finding { + entity_type: "PERSON".to_owned(), + matched_text: "José".to_owned(), + byte_range: TextRange { start: 5, end: 10 }, + codepoint_range: TextRange { start: 5, end: 9 }, + confidence: None, + detector_name: "test".to_owned(), + detector_version: None, + }; + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_exact_allowlist("PERSON", vec!["José".to_owned()]) + .unwrap(); + + let result = transform(text, &[finding], &config).unwrap(); + + assert_eq!(result.text, text); + assert!(result.transformations.is_empty()); + } + + #[test] + fn scan_and_transform_uses_the_divided_configuration_envelope() { + let config = parse_scan_and_transform_config(&json!({ + "scan": { "locale": "en-US" }, + "transform": { + "default": { "strategy": "redact" }, + "entities": ["EMAIL"] + } + })) + .unwrap(); + + assert_eq!(config.scan_config().locale(), Some("en-US")); + assert_eq!( + scan_and_transform("Email jane@example.com", &config) + .unwrap() + .text, + "Email [EMAIL]" + ); + + let error = parse_scan_and_transform_config(&json!({ + "transform": { "default": { "strategy": "redact", "extra": true } } + })) + .unwrap_err(); + assert_eq!(error.path(), Some("/transform/default/extra")); + } + + #[test] + fn valid_configuration_for_unselected_entities_remains_dormant() { + let config = parse_transformation_config(&json!({ + "default": { "strategy": "redact" }, + "entities": ["EMAIL"], + "overrides": { "PHONE": { "strategy": "remove" } }, + "allow": { + "exact": { "PERSON": ["Jane Example"] }, + "regex": { + "CUSTOM": [{ "pattern": "value-[0-9]+" }] + } + } + })) + .unwrap(); + + let text = "Email jane@example.com or call (212) 555-0100"; + let result = transform(text, &scan(text), &config).unwrap(); + + assert_eq!(result.text, "Email [EMAIL] or call (212) 555-0100"); + assert_eq!(result.transformations.len(), 1); + } + + #[test] + fn regex_allowlist_limits_reject_the_complete_configuration() { + let too_many = (0..=MAX_REGEX_RULES) + .map(|index| RegexAllowRule::new(format!("value-{index}"), true)) + .collect(); + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("CUSTOM", too_many) + .unwrap_err(); + assert_eq!(error.reason(), Some(PrivacyErrorReason::LimitExceeded)); + assert_eq!(error.path(), Some("/allow/regex")); + + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist( + "CUSTOM", + vec![RegexAllowRule::new( + "x".repeat(MAX_REGEX_PATTERN_BYTES + 1), + true, + )], + ) + .unwrap_err(); + assert_eq!(error.reason(), Some(PrivacyErrorReason::LimitExceeded)); + + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("CUSTOM", vec![RegexAllowRule::new("(", true)]) + .unwrap_err(); + assert_eq!(error.reason(), Some(PrivacyErrorReason::InvalidRegex)); + + let aggregate = (0..11) + .map(|index| RegexAllowRule::new(format!("{}-{index}", "x".repeat(950)), true)) + .collect(); + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("CUSTOM", aggregate) + .unwrap_err(); + assert_eq!(error.reason(), Some(PrivacyErrorReason::LimitExceeded)); + assert_eq!(error.path(), Some("/allow/regex")); + + let error = TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("CUSTOM", vec![RegexAllowRule::new(r"\w{1000}", true)]) + .unwrap_err(); + assert_eq!(error.reason(), Some(PrivacyErrorReason::LimitExceeded)); + } + #[test] fn fully_masks_every_codepoint_including_punctuation() { let text = "Email jane@example.com"; let findings = scan(text); let strategy = TransformationStrategy::Mask(MaskConfig::default()); - let result = transform(text, &findings, strategy).unwrap(); + let result = transform(text, &findings, &config(strategy)).unwrap(); assert_eq!(result.text, "Email ****************"); assert_eq!(result.transformations[0].strategy, strategy); @@ -1436,11 +2588,15 @@ mod tests { TransformationStrategy::Mask(MaskConfig::new('*', MaskReveal::Last(4)).unwrap()); assert_eq!( - transform(text, &findings, reveal_first).unwrap().text, + transform(text, &findings, &config(reveal_first)) + .unwrap() + .text, "Email jane************" ); assert_eq!( - transform(text, &findings, reveal_last).unwrap().text, + transform(text, &findings, &config(reveal_last)) + .unwrap() + .text, "Email ************.com" ); } @@ -1456,10 +2612,17 @@ mod tests { ); assert_eq!( - transform(text, &findings, reveal_none).unwrap().text, + transform(text, &findings, &config(reveal_none)) + .unwrap() + .text, "Email ****************" ); - assert_eq!(transform(text, &findings, reveal_all).unwrap().text, text); + assert_eq!( + transform(text, &findings, &config(reveal_all)) + .unwrap() + .text, + text + ); } #[test] @@ -1488,7 +2651,7 @@ mod tests { let strategy = TransformationStrategy::Mask(MaskConfig::new('•', MaskReveal::None).unwrap()); - let result = transform(text, &[finding], strategy).unwrap(); + let result = transform(text, &[finding], &config(strategy)).unwrap(); assert_eq!(result.text, "A •• Z"); assert_eq!( @@ -1506,7 +2669,7 @@ mod tests { let text = "Email jane@example.com today"; let findings = scan(text); - let result = transform(text, &findings, TransformationStrategy::Remove).unwrap(); + let result = transform(text, &findings, &config(TransformationStrategy::Remove)).unwrap(); assert_eq!(result.text, "Email today"); assert_eq!(result.transformations[0].replacement, ""); @@ -1527,11 +2690,11 @@ mod tests { findings[0].matched_text = "other@example.com".to_owned(); assert_eq!( - transform(text, &findings, TransformationStrategy::Redact), - Err(TransformError { - finding_index: 0, - kind: FindingValidationError::MatchedTextMismatch, - }) + transform(text, &findings, &config(TransformationStrategy::Redact)), + Err(PrivacyError::invalid_finding( + 0, + FindingValidationError::MatchedTextMismatch, + )) ); } @@ -1608,11 +2771,8 @@ mod tests { for (finding, expected_kind) in cases { assert_eq!( - transform(text, &[finding], TransformationStrategy::Redact), - Err(TransformError { - finding_index: 0, - kind: expected_kind, - }) + transform(text, &[finding], &config(TransformationStrategy::Redact)), + Err(PrivacyError::invalid_finding(0, expected_kind)) ); } } @@ -1630,7 +2790,7 @@ mod tests { let result = transform( text, &[lower_confidence, higher_confidence.clone()], - TransformationStrategy::Redact, + &config(TransformationStrategy::Redact), ) .unwrap(); @@ -1655,7 +2815,7 @@ mod tests { let result = transform( text, &[inner, outer.clone()], - TransformationStrategy::Redact, + &config(TransformationStrategy::Redact), ) .unwrap(); @@ -1668,7 +2828,8 @@ mod tests { fn scan_and_transform_redacts_unicode_input_with_exact_output_ranges() { let text = "👋 jane@example.com and jane@example.com"; - let result = scan_and_transform(text, TransformationStrategy::Redact).unwrap(); + let config = ScanAndTransformConfig::new(config(TransformationStrategy::Redact)); + let result = scan_and_transform(text, &config).unwrap(); assert_eq!(result.text, "👋 [EMAIL] and [EMAIL]"); assert_eq!(result.transformations.len(), 2); @@ -1698,7 +2859,7 @@ mod tests { let result = transform( text, &[lower, higher.clone()], - TransformationStrategy::Redact, + &config(TransformationStrategy::Redact), ) .unwrap(); @@ -1715,7 +2876,7 @@ mod tests { let result = transform( text, &[scored, unscored.clone()], - TransformationStrategy::Redact, + &config(TransformationStrategy::Redact), ) .unwrap(); @@ -1732,7 +2893,7 @@ mod tests { let result = transform( text, &[later, earlier.clone()], - TransformationStrategy::Redact, + &config(TransformationStrategy::Redact), ) .unwrap(); @@ -1742,7 +2903,7 @@ mod tests { #[test] fn empty_findings_leave_text_unchanged() { - let result = transform("plain text", &[], TransformationStrategy::Redact).unwrap(); + let result = transform("plain text", &[], &config(TransformationStrategy::Redact)).unwrap(); assert_eq!(result.text, "plain text"); assert!(result.transformations.is_empty()); diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index a2e7178..17d62cf 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -53,6 +53,185 @@ with typed enum variants; object-oriented bindings serialize it as: Fields that do not belong to the selected strategy are rejected rather than silently ignored. +### Transformation selection + +Transformation configuration has one required default strategy and may add an +entity selection, per-entity strategy overrides, and entity-scoped exact or +regex allowlists. Entity names are matched exactly and case-sensitively. The +public serialized shape is: + +```text +{ + default: { strategy: "redact" }, + entities: ["EMAIL", "PHONE"], + overrides: { + PHONE: { + strategy: "mask", + reveal: { direction: "last", count: 4 } + } + }, + allow: { + exact: { EMAIL: ["support@example.com"] }, + regex: { + EMAIL: [ + { pattern: ".*@example\\.com", case_sensitive: true } + ] + } + } +} +``` + +This envelope replaces the temporary single-strategy input introduced while +the transformation framework was built. The old `{ strategy: ... }` shape is +not retained as a shorthand or exposed through a second operation. A caller +that needs only one strategy supplies it as `default`. Rust accepts the typed +transformation configuration, and Python, Node, and WASM accept the same +serialized envelope. Transformation records continue to report the strategy +actually applied to each finding rather than copying the request envelope. + +Omitting `entities` selects all supplied findings. A non-empty list selects +only those entity types. An explicitly empty list is rejected, as are duplicate +entity names. Entity types remain extensible and are not limited to built-in +detectors. + +Exact allowlists compare against the complete `matched_text` and are +case-sensitive. Regex allowlists use full-match semantics and are +case-sensitive unless explicitly configured otherwise. Patterns are compiled +once per configuration using a non-backtracking regex engine and are subject to +the following limits: + +- at most 100 regex rules per transformation configuration after + deduplication; +- at most 1 KiB of UTF-8 source per pattern; +- at most 10 KiB of aggregate UTF-8 regex source; and +- at most 1 MiB of compiled representation per compiled pattern group. + +Invalid patterns and limit violations reject the complete configuration. No +pattern is silently dropped and transformation never proceeds with a partial +allowlist. A runtime timeout is unnecessary because the selected regex engine +does not use backtracking. Valid broad expressions such as `.*` are accepted as +intentional caller policy rather than rejected based on inferred intent. Empty +per-entity allowlists are rejected; repeated allow values and identical regex +rules are deduplicated. + +Every supplied configuration entry is validated even when its entity type is +not selected by the current invocation. Valid overrides and allowlists for +unselected entity types are dormant: they have no effect, but do not make the +configuration invalid. This permits one reusable privacy profile to define +behavior for all relevant entity types while individual calls select a subset. +Configuration is never validated against only the findings present in one +document. + +Empty optional structural objects are accepted as equivalent to omission. This +includes empty `overrides` and `allow` objects, empty `exact` and `regex` maps, +and an empty `scan` object. `TransformationConfig` itself is not optional and +still requires `default`. This supports configuration builders and serializers +without giving an empty wrapper object a hidden policy meaning. + +Semantic empty values are rejected. These include an empty `entities` list, +empty per-entity allowlist arrays, empty or whitespace-only entity names, empty +exact allowlist values, and empty regex pattern source. Explicit `null` is not +an alias for omission and is rejected for every optional field. Unknown fields +are rejected recursively at every configuration level so misspelled or stale +policy fields cannot be silently ignored. + +Processing order is: + +1. validate all supplied findings and the complete configuration; +2. filter findings using `entities`; +3. exempt exact and regex allowlist matches; +4. resolve duplicates and overlaps among the remaining findings; +5. choose an entity-specific override when present, otherwise `default`; and +6. apply transformations in source document order. + +Allowlisted findings remain unchanged and produce no transformation record. +Findings excluded by entity selection likewise remain unchanged and produce no +record. Locale is scanning configuration because it affects detection; it is +not part of transformation selection for caller-supplied findings. + +Configurable entity-type priority for overlap resolution is deferred. Slice 4 +retains the equal-priority deterministic ranking defined in this ADR. + +### Scan-and-transform configuration + +Scanning and transformation use separate reusable configuration types because +they control different layers. `ScanConfig` controls detection concerns such as +locale and future detector settings. `TransformationConfig` controls selection, +exemptions, and replacement of supplied findings. + +`scan_and_transform` accepts one explicitly divided envelope: + +```text +{ + scan: { + locale: "en-US" + }, + transform: { + default: { strategy: "redact" }, + entities: ["EMAIL"] + } +} +``` + +The `transform` member is required. The `scan` member may be omitted to use +scanner defaults. Unknown scanning fields are rejected. Locale and other +detection settings never appear in `TransformationConfig` and cannot affect +the transformation of caller-supplied findings. + +The same transformation configuration can therefore be passed directly to +`transform` or nested under `transform` in `scan_and_transform`. Entity +selection remains a transformation rule. `scan_and_transform` may avoid +unnecessary detector work when doing so is provably equivalent, but such an +optimization must not change the findings retained or the transformation +result. + +### Error contract + +Top-level operations expose one structured error contract across Rust, Python, +Node, and WASM. The stable top-level error codes are: + +```text +invalid_configuration +invalid_finding +internal_error +``` + +Caller-correctable errors also include a stable machine-readable `reason` and +an RFC 6901 JSON Pointer `path` identifying the invalid request location. +Finding errors additionally include `finding_index`. Initial reason values +include: + +```text +missing_field +unknown_field +invalid_type +invalid_value +empty_value +duplicate_value +invalid_regex +limit_exceeded +matched_text_mismatch +inconsistent_ranges +out_of_bounds +invalid_boundary +invalid_confidence +``` + +Codes, reasons, and paths are public API. Human-readable messages are intended +for diagnostics and are not stable or suitable for programmatic parsing. Error +details never contain source text, matched PII, or other sensitive input +values. Validation is atomic and no partial transformation result accompanies +an error. + +Rust represents the contract with a typed `PrivacyError`. Python exposes +configuration and finding errors as `ValueError` subclasses and internal +failures as a `RuntimeError` subclass. Node and WASM expose JavaScript +`DataFogError` objects with `code`, `reason`, `path`, and optional +`findingIndex`; WASM never rejects with a bare string. Each binding may use its +native exception hierarchy, but the canonical fields retain the same meaning. +Additional reason values may be introduced as validation expands without +creating new top-level categories for every validation case. + `allow`, `warn`, and `block` are governance decisions, not Core operations or transformation strategies. A separate governance layer may consume findings and transformation results to make those decisions. The calling application, diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 60a866f..a462c24 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -83,14 +83,43 @@ replacements. Invalid strategy fields and masking characters are rejected. ## Slice 4: Transformation selection -- Add entity-type selection. -- Add exact, case-sensitive allowlists. -- Add full-match regex allowlists with bounded behavior. -- Add locale selection and per-entity strategy overrides. -- Define transformation-configuration validation and precedence. - -**Proof:** the same configuration retains the same findings whether it scans -internally or receives precomputed findings. +- Replace the temporary single-strategy request with one canonical + transformation-configuration envelope. Do not retain the old shape as a + shorthand or add a second transformation operation. +- Add an explicit default strategy with exact, case-sensitive per-entity + overrides. +- Treat an omitted entity selection as all supplied findings, a non-empty + selection as an exact case-sensitive filter, and an empty selection as an + error. +- Add entity-scoped exact, case-sensitive allowlists. +- Add entity-scoped, full-match regex allowlists with case-sensitive matching + by default. Limit each configuration to 100 deduplicated rules, 1 KiB per + pattern, 10 KiB aggregate source, and 1 MiB per compiled pattern group; + reject invalid or over-limit configurations atomically. +- Validate every supplied configuration entry, including entries for entity + types not selected by the current call. Valid but unselected overrides and + allowlists remain dormant rather than making the configuration invalid. +- Accept empty structural objects and maps as omission, while rejecting empty + semantic values, explicit `null`, and unknown fields at every nesting level. +- Apply configuration in this order: validate, select entities, apply + allowlists, resolve overlaps, choose the entity override or default strategy, + and transform in document order. +- Keep locale in scanning configuration because it affects detection, not the + transformation of supplied findings. +- Give `scan_and_transform` one divided request envelope with an optional + `scan` configuration and required `transform` configuration. Reuse those + configuration types in the standalone operations. +- Retain equal entity-type priority during overlap resolution; configurable + entity priorities are deferred beyond Slice 4. +- Standardize atomic structured errors across bindings using stable + `invalid_configuration`, `invalid_finding`, and `internal_error` codes, + machine-readable reasons, and RFC 6901 request paths. Never include sensitive + input values in errors. + +**Proof:** selection, exact and regex exemptions, overlap interactions, +override fallback, dormant rules, malformed configuration, and Unicode cases +produce the same result whether findings come directly from `scan` or are +supplied to `transform`. ## Slice 5: Compatibility hash diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index ed20062..2d63f55 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -30,7 +30,7 @@ function writeConsumerTest() { import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { scan, scanAndTransform, transform } from "@datafog/node"; +import { DataFogError, scan, scanAndTransform, transform } from "@datafog/node"; const fixturesDirectory = process.argv[2]; @@ -82,10 +82,18 @@ for (const name of ["development.jsonl", "final.jsonl"]) { const emojiFinding = scan("👋 jane@example.com")[0]; assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); +assert.deepEqual( + scan("Email jane@example.com", { locale: "en-US" }), + scan("Email jane@example.com"), +); const transformText = "👋 jane@example.com and jane@example.com"; -const explicit = transform(transformText, scan(transformText), { strategy: "redact" }); -const convenience = scanAndTransform(transformText, { strategy: "redact" }); +const explicit = transform(transformText, scan(transformText), { + default: { strategy: "redact" }, +}); +const convenience = scanAndTransform(transformText, { + transform: { default: { strategy: "redact" } }, +}); assert.deepEqual(explicit, convenience); assert.equal(explicit.text, "👋 [EMAIL] and [EMAIL]"); assert.equal(explicit.transformations.length, 2); @@ -93,13 +101,19 @@ assert.equal(explicit.transformations[0].replacement, "[EMAIL]"); assert.deepEqual(explicit.transformations[0].outputByteRange, { start: 5, end: 12 }); assert.deepEqual(explicit.transformations[0].outputCodepointRange, { start: 2, end: 9 }); assert.equal( - scanAndTransform("Email jane@example.com", { strategy: "mask" }).text, + scanAndTransform("Email jane@example.com", { + transform: { default: { strategy: "mask" } }, + }).text, "Email ****************", ); const partialMask = scanAndTransform("Email jane@example.com", { - strategy: "mask", - character: "•", - reveal: { direction: "last", count: 4 }, + transform: { + default: { + strategy: "mask", + character: "•", + reveal: { direction: "last", count: 4 }, + }, + }, }); assert.equal(partialMask.text, "Email ••••••••••••.com"); assert.equal(partialMask.transformations[0].strategy, "mask"); @@ -108,13 +122,19 @@ assert.deepEqual(partialMask.transformations[0].outputByteRange, { start: 6, end assert.equal( scanAndTransform("Email jane@example.com", { - strategy: "mask", - reveal: { direction: "first", count: 99 }, + transform: { + default: { + strategy: "mask", + reveal: { direction: "first", count: 99 }, + }, + }, }).text, "Email jane@example.com", ); -const removed = scanAndTransform("Email jane@example.com today", { strategy: "remove" }); +const removed = scanAndTransform("Email jane@example.com today", { + transform: { default: { strategy: "remove" } }, +}); assert.equal(removed.text, "Email today"); assert.equal(removed.transformations[0].strategy, "remove"); assert.equal(removed.transformations[0].replacement, ""); @@ -130,17 +150,58 @@ for (const invalidConfig of [ { strategy: "mask", reveal: { direction: "middle", count: 4 } }, ]) { assert.throws( - () => scanAndTransform("Email jane@example.com", invalidConfig), - TypeError, + () => scanAndTransform("Email jane@example.com", { + transform: { default: invalidConfig }, + }), + DataFogError, ); } assert.throws( () => transform( transformText, [{ ...scan(transformText)[0], confidence: 2 }], - { strategy: "redact" }, + { default: { strategy: "redact" } }, ), - /InvalidConfidence/, + (error) => + error instanceof DataFogError && + error.code === "invalid_finding" && + error.reason === "invalid_confidence" && + error.path === "/findings/0/confidence" && + error.findingIndex === 0, +); + +const selected = scanAndTransform( + "Email support@example.com or call (212) 555-0100", + { + scan: { locale: "en-US" }, + transform: { + default: { strategy: "redact" }, + entities: ["EMAIL", "PHONE"], + overrides: { + PHONE: { + strategy: "mask", + reveal: { direction: "last", count: 4 }, + }, + }, + allow: { + exact: { EMAIL: ["support@example.com"] }, + regex: {}, + }, + }, + }, +); +assert.equal(selected.text, "Email support@example.com or call **********0100"); +assert.equal(selected.transformations.length, 1); + +assert.throws( + () => transform(transformText, scan(transformText), { + default: { strategy: "redact" }, + overides: {}, + }), + (error) => + error instanceof DataFogError && + error.reason === "unknown_field" && + error.path === "/overides", ); console.log("Installed @datafog/node package matches fixtures and transform contracts."); @@ -157,6 +218,7 @@ import { type EntityType, type Finding, type MaskRevealConfig, + type ScanAndTransformConfig, type TextRange, type TransformationConfig, type TransformResult, @@ -168,19 +230,22 @@ const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; const explicit: TransformResult = transform( "Email jane@example.com", findings, - { strategy: "redact" }, + { default: { strategy: "redact" } }, ); const convenience: TransformResult = scanAndTransform( "Email jane@example.com", - { strategy: "redact" }, + { transform: { default: { strategy: "redact" } } }, ); const reveal: MaskRevealConfig = { direction: "last", count: 4 }; const maskConfig: TransformationConfig = { - strategy: "mask", - character: "•", - reveal, + default: { + strategy: "mask", + character: "•", + reveal, + }, }; -const masked: TransformResult = scanAndTransform("Email jane@example.com", maskConfig); +const combined: ScanAndTransformConfig = { transform: maskConfig }; +const masked: TransformResult = scanAndTransform("Email jane@example.com", combined); void entityType; void range; diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index 9db6ec0..fb79ea1 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -86,6 +86,7 @@ import { type EntityType, type Finding, type MaskRevealConfig, + type ScanAndTransformConfig, type TextRange, type TransformationConfig, type TransformResult, @@ -98,19 +99,22 @@ const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; const transformed: TransformResult = transform( "Email jane@example.com", findings, - { strategy: "redact" }, + { default: { strategy: "redact" } }, ); const scannedAndTransformed: TransformResult = scanAndTransform( "Email jane@example.com", - { strategy: "redact" }, + { transform: { default: { strategy: "redact" } } }, ); const reveal: MaskRevealConfig = { direction: "last", count: 4 }; const maskConfig: TransformationConfig = { - strategy: "mask", - character: "•", - reveal, + default: { + strategy: "mask", + character: "•", + reveal, + }, }; -const masked: TransformResult = scanAndTransform("Email jane@example.com", maskConfig); +const combined: ScanAndTransformConfig = { transform: maskConfig }; +const masked: TransformResult = scanAndTransform("Email jane@example.com", combined); void ready; void entityType; @@ -191,7 +195,7 @@ try { await page.goto(serverInfo.url); await page.evaluate(async () => { - const { init, scan, scanAndTransform, transform } = await import( + const { DataFogError, init, scan, scanAndTransform, transform } = await import( "/node_modules/@datafog/wasm/index.js" ); @@ -268,11 +272,21 @@ try { ) { throw new Error("Unicode ranges do not use the documented coordinate systems"); } + if ( + JSON.stringify(scan("Email jane@example.com", { locale: "en-US" })) !== + JSON.stringify(scan("Email jane@example.com")) + ) { + throw new Error("standalone scan configuration changed detector output"); + } const text = "👋 jane@example.com and jane@example.com"; const findings = scan(text); - const explicit = transform(text, findings, { strategy: "redact" }); - const convenient = scanAndTransform(text, { strategy: "redact" }); + const explicit = transform(text, findings, { + default: { strategy: "redact" }, + }); + const convenient = scanAndTransform(text, { + transform: { default: { strategy: "redact" } }, + }); if (JSON.stringify(explicit) !== JSON.stringify(convenient)) { throw new Error("explicit and convenience transforms differ"); } @@ -294,16 +308,22 @@ try { } if ( - scanAndTransform("Email jane@example.com", { strategy: "mask" }).text !== + scanAndTransform("Email jane@example.com", { + transform: { default: { strategy: "mask" } }, + }).text !== "Email ****************" ) { throw new Error("full masking did not replace every code point"); } const partialMask = scanAndTransform("Email jane@example.com", { - strategy: "mask", - character: "•", - reveal: { direction: "last", count: 4 }, + transform: { + default: { + strategy: "mask", + character: "•", + reveal: { direction: "last", count: 4 }, + }, + }, }); if ( partialMask.text !== "Email ••••••••••••.com" || @@ -317,15 +337,19 @@ try { if ( scanAndTransform("Email jane@example.com", { - strategy: "mask", - reveal: { direction: "first", count: 99 }, + transform: { + default: { + strategy: "mask", + reveal: { direction: "first", count: 99 }, + }, + }, }).text !== "Email jane@example.com" ) { throw new Error("oversized reveal count should preserve the finding"); } const removed = scanAndTransform("Email jane@example.com today", { - strategy: "remove", + transform: { default: { strategy: "remove" } }, }); if ( removed.text !== "Email today" || @@ -347,20 +371,76 @@ try { { strategy: "mask", reveal: { direction: "middle", count: 4 } }, ]) { expectThrows( - () => scanAndTransform("Email jane@example.com", invalidConfig), - "TypeError", + () => + scanAndTransform("Email jane@example.com", { + transform: { default: invalidConfig }, + }), + "DataFogError", + ); + } + + try { + transform( + text, + [{ ...findings[0], confidence: 2 }], + { default: { strategy: "redact" } }, ); + throw new Error("invalid finding should fail"); + } catch (error) { + if ( + !(error instanceof DataFogError) || + error.code !== "invalid_finding" || + error.reason !== "invalid_confidence" || + error.path !== "/findings/0/confidence" || + error.findingIndex !== 0 + ) { + throw error; + } } - expectThrows( - () => - transform( - text, - [{ ...findings[0], confidence: 2 }], - { strategy: "redact" }, - ), - "Error", + const selected = scanAndTransform( + "Email support@example.com or call (212) 555-0100", + { + scan: { locale: "en-US" }, + transform: { + default: { strategy: "redact" }, + entities: ["EMAIL", "PHONE"], + overrides: { + PHONE: { + strategy: "mask", + reveal: { direction: "last", count: 4 }, + }, + }, + allow: { + exact: { EMAIL: ["support@example.com"] }, + regex: {}, + }, + }, + }, ); + if ( + selected.text !== "Email support@example.com or call **********0100" || + selected.transformations.length !== 1 + ) { + throw new Error("selection, overrides, or allowlists failed"); + } + + try { + transform(text, findings, { + default: { strategy: "redact" }, + overides: {}, + }); + throw new Error("unknown configuration field should fail"); + } catch (error) { + if ( + !(error instanceof DataFogError) || + error.code !== "invalid_configuration" || + error.reason !== "unknown_field" || + error.path !== "/overides" + ) { + throw error; + } + } }); console.log("Installed @datafog/wasm package matches fixtures and the Finding contract.");