diff --git a/README.md b/README.md index c016a1a..1424a71 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ optional confidence, detector name, optional detector version Both ranges use zero-based, end-exclusive offsets. The byte range addresses the UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based -detectors currently report no confidence score. +detectors currently report no confidence score. Node.js and browser WASM also +return an explicitly named UTF-16 code-unit range that can be passed directly +to JavaScript `String.prototype.slice`. The transformation strategies are `redact`, `mask`, `remove`, `pseudonymize`, and `tokenize` in Rust, Python, and Node.js. @@ -26,8 +28,9 @@ unsupported in browser WASM. `scanAndTransform` in JavaScript) is the explicit scan-then-transform convenience. Results include the transformed text and an ordered record for every applied replacement, including its source metadata and output byte and -code-point ranges. Transformation records never include the original matched -text. +code-point ranges. Node.js and browser WASM additionally return source and +output UTF-16 ranges. Transformation records never include the original +matched text. Transformation calls require an envelope with a default strategy. It can also select entity types, override the strategy per entity, and exempt exact or diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index 14cebfb..d6402d4 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -141,7 +141,7 @@ export declare class PrivacyManager { constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, - findings: Finding[], + findings: FindingInput[], config: TransformationConfig, context?: PrivacyContext, ): Promise; diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 75b5452..b39558e 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -141,7 +141,7 @@ export declare class PrivacyManager { constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, - findings: Finding[], + findings: FindingInput[], config: TransformationConfig, context?: PrivacyContext, ): Promise; @@ -157,11 +157,22 @@ export interface Finding { readonly matchedText: string readonly byteRange: TextRange readonly codepointRange: TextRange + readonly utf16Range: TextRange readonly confidence?: number readonly detectorName: string readonly detectorVersion?: string } +export interface FindingInput { + entityType: EntityType + matchedText: string + byteRange: TextRange + codepointRange: TextRange + confidence?: number + detectorName: string + detectorVersion?: string +} + export interface KeySelector { readonly index: number readonly keyRef: string @@ -176,11 +187,11 @@ export interface PreparedScanAndTransform { export declare function prepareScanAndTransform(text: string, config: ScanAndTransformConfig): PreparedScanAndTransform -export declare function requiredKeySelectors(text: string, findings: Array, config: TransformationConfig): Array +export declare function requiredKeySelectors(text: string, findings: FindingInput[], config: TransformationConfig): Array export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array -export declare function requiredTokenizationItems(text: string, findings: Array, config: TransformationConfig, context?: PrivacyContext | undefined): Array +export declare function requiredTokenizationItems(text: string, findings: FindingInput[], config: TransformationConfig, context?: PrivacyContext | undefined): Array export interface ResolvedKeyInput { selectorIndex: number @@ -191,8 +202,10 @@ export interface ResolvedKeyInput { export interface Restoration { readonly sourceByteRange: TextRange readonly sourceCodepointRange: TextRange + readonly sourceUtf16Range: TextRange readonly outputByteRange: TextRange readonly outputCodepointRange: TextRange + readonly outputUtf16Range: TextRange readonly tokenRef: string readonly resolvedTokenVersion: string } @@ -240,12 +253,13 @@ export interface TokenizeResultInput { } /** Transform explicit findings without scanning implicitly. */ -export declare function transform(text: string, findings: Array, config: TransformationConfig): TransformResult +export declare function transform(text: string, findings: FindingInput[], config: TransformationConfig): TransformResult export interface Transformation { readonly entityType: string readonly sourceByteRange: TextRange readonly sourceCodepointRange: TextRange + readonly sourceUtf16Range: TextRange readonly confidence?: number readonly detectorName: string readonly detectorVersion?: string @@ -253,6 +267,7 @@ export interface Transformation { readonly replacement: string readonly outputByteRange: TextRange readonly outputCodepointRange: TextRange + readonly outputUtf16Range: TextRange readonly keyRef?: string readonly resolvedKeyVersion?: string readonly tokenRef?: string @@ -264,6 +279,6 @@ export interface TransformResult { readonly transformations: Array } -export declare function transformWithProviderResults(text: string, findings: Array, config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array, tokenResults: Array): TransformResult +export declare function transformWithProviderResults(text: string, findings: FindingInput[], config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array, tokenResults: Array): TransformResult -export declare function transformWithResolvedKeys(text: string, findings: Array, config: TransformationConfig, resolvedKeys: Array): TransformResult +export declare function transformWithResolvedKeys(text: string, findings: FindingInput[], config: TransformationConfig, resolvedKeys: Array): TransformResult diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 2ebbe27..e511233 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -27,6 +27,9 @@ pub struct Finding { #[napi(readonly)] pub codepoint_range: TextRange, + #[napi(readonly)] + pub utf16_range: TextRange, + #[napi(readonly)] pub confidence: Option, @@ -37,6 +40,18 @@ pub struct Finding { pub detector_version: Option, } +#[napi(object, object_to_js = false)] +pub struct FindingInput { + #[napi(ts_type = "EntityType")] + pub entity_type: String, + pub matched_text: String, + pub byte_range: TextRange, + pub codepoint_range: TextRange, + pub confidence: Option, + pub detector_name: String, + pub detector_version: Option, +} + #[napi(object, object_from_js = false)] pub struct Transformation { #[napi(readonly)] @@ -48,6 +63,9 @@ pub struct Transformation { #[napi(readonly)] pub source_codepoint_range: TextRange, + #[napi(readonly)] + pub source_utf16_range: TextRange, + #[napi(readonly)] pub confidence: Option, @@ -69,6 +87,9 @@ pub struct Transformation { #[napi(readonly)] pub output_codepoint_range: TextRange, + #[napi(readonly)] + pub output_utf16_range: TextRange, + #[napi(readonly)] pub key_ref: Option, @@ -157,10 +178,14 @@ pub struct Restoration { #[napi(readonly)] pub source_codepoint_range: TextRange, #[napi(readonly)] + pub source_utf16_range: TextRange, + #[napi(readonly)] pub output_byte_range: TextRange, #[napi(readonly)] pub output_codepoint_range: TextRange, #[napi(readonly)] + pub output_utf16_range: TextRange, + #[napi(readonly)] pub token_ref: String, #[napi(readonly)] pub resolved_token_version: String, @@ -199,19 +224,26 @@ fn js_range(range: datafog_core::TextRange) -> napi::Result { }) } -fn js_finding(finding: datafog_core::Finding) -> napi::Result { +fn js_utf16_range(text: &str, range: datafog_core::TextRange) -> napi::Result { + datafog_core::utf16_range(text, range) + .map_err(|error| Error::new(Status::GenericFailure, error.to_string())) + .and_then(js_range) +} + +fn js_finding(text: &str, finding: datafog_core::Finding) -> napi::Result { Ok(Finding { entity_type: finding.entity_type, matched_text: finding.matched_text, byte_range: js_range(finding.byte_range)?, codepoint_range: js_range(finding.codepoint_range)?, + utf16_range: js_utf16_range(text, finding.byte_range)?, confidence: finding.confidence.map(f64::from), detector_name: finding.detector_name, detector_version: finding.detector_version, }) } -fn core_finding(finding: Finding) -> datafog_core::Finding { +fn core_finding(finding: FindingInput) -> datafog_core::Finding { datafog_core::Finding { entity_type: finding.entity_type, matched_text: finding.matched_text, @@ -246,9 +278,12 @@ fn js_privacy_error(error: datafog_core::PrivacyError) -> Error { Error::new(status, payload.to_string()) } -fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result { +fn js_transform_result( + source_text: &str, + result: datafog_core::TransformResult, +) -> napi::Result { + let output_text = &result.text; Ok(TransformResult { - text: result.text, transformations: result .transformations .into_iter() @@ -257,6 +292,10 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result napi::Result napi::Result>>()?, + text: result.text, }) } @@ -295,9 +339,12 @@ fn core_token_results(results: Vec) -> Vec napi::Result { +fn js_restore_result( + source_text: &str, + result: datafog_core::RestoreResult, +) -> napi::Result { + let output_text = &result.text; Ok(RestoreResult { - text: result.text, restorations: result .restorations .into_iter() @@ -305,13 +352,16 @@ fn js_restore_result(result: datafog_core::RestoreResult) -> napi::Result>>()?, + text: result.text, }) } @@ -366,7 +416,7 @@ pub fn scan( }; datafog_core::scan_with_config(&text, &config) .into_iter() - .map(js_finding) + .map(|finding| js_finding(&text, finding)) .collect() } @@ -375,7 +425,7 @@ pub fn scan( pub fn transform( env: Env, text: String, - findings: Vec, + #[napi(ts_arg_type = "FindingInput[]")] findings: Vec, #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, ) -> napi::Result { let config: serde_json::Value = env.from_js_value(config)?; @@ -383,7 +433,7 @@ pub fn transform( let findings = findings.into_iter().map(core_finding).collect::>(); datafog_core::transform(&text, &findings, &config) .map_err(js_privacy_error) - .and_then(js_transform_result) + .and_then(|result| js_transform_result(&text, result)) } /// Scan text and transform the detected findings. @@ -398,14 +448,14 @@ pub fn scan_and_transform( datafog_core::parse_scan_and_transform_config(&config).map_err(js_privacy_error)?; datafog_core::scan_and_transform(&text, &config) .map_err(js_privacy_error) - .and_then(js_transform_result) + .and_then(|result| js_transform_result(&text, result)) } #[napi(strict, catch_unwind)] pub fn required_key_selectors( env: Env, text: String, - findings: Vec, + #[napi(ts_arg_type = "FindingInput[]")] findings: Vec, #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, ) -> napi::Result> { let config: serde_json::Value = env.from_js_value(config)?; @@ -420,7 +470,7 @@ pub fn required_key_selectors( pub fn transform_with_resolved_keys( env: Env, text: String, - findings: Vec, + #[napi(ts_arg_type = "FindingInput[]")] findings: Vec, #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, resolved_keys: Vec, ) -> napi::Result { @@ -432,7 +482,7 @@ pub fn transform_with_resolved_keys( let bindings = core_key_bindings(selectors, resolved_keys)?; datafog_core::transform_with_resolved_keys(&text, &findings, &config, bindings) .map_err(js_privacy_error) - .and_then(js_transform_result) + .and_then(|result| js_transform_result(&text, result)) } #[napi(strict, catch_unwind)] @@ -451,7 +501,7 @@ pub fn prepare_scan_and_transform( Ok(PreparedScanAndTransform { findings: findings .into_iter() - .map(js_finding) + .map(|finding| js_finding(&text, finding)) .collect::>>()?, selectors: js_key_selectors(&selectors)?, }) @@ -461,7 +511,7 @@ pub fn prepare_scan_and_transform( pub fn required_tokenization_items( env: Env, text: String, - findings: Vec, + #[napi(ts_arg_type = "FindingInput[]")] findings: Vec, #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, ) -> napi::Result> { @@ -493,7 +543,7 @@ pub fn required_tokenization_items( pub fn transform_with_provider_results( env: Env, text: String, - findings: Vec, + #[napi(ts_arg_type = "FindingInput[]")] findings: Vec, #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, resolved_keys: Vec, @@ -521,7 +571,7 @@ pub fn transform_with_provider_results( core_token_results(token_results), ) .map_err(js_privacy_error) - .and_then(js_transform_result) + .and_then(|result| js_transform_result(&text, result)) } #[napi(strict, catch_unwind)] @@ -562,5 +612,5 @@ pub fn restore_with_results( .collect(); datafog_core::restore_with_results(&text, &context, results) .map_err(js_privacy_error) - .and_then(js_restore_result) + .and_then(|result| js_restore_result(&text, result)) } diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index 32b95f7..5833f86 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -76,7 +76,7 @@ export interface TextRange { readonly end: number; } -export interface Finding { +export interface FindingInput { readonly entityType: EntityType; readonly matchedText: string; readonly byteRange: TextRange; @@ -86,10 +86,15 @@ export interface Finding { readonly detectorVersion?: string; } +export interface Finding extends FindingInput { + readonly utf16Range: TextRange; +} + export interface Transformation { readonly entityType: EntityType; readonly sourceByteRange: TextRange; readonly sourceCodepointRange: TextRange; + readonly sourceUtf16Range: TextRange; readonly confidence?: number; readonly detectorName: string; readonly detectorVersion?: string; @@ -97,6 +102,7 @@ export interface Transformation { readonly replacement: string; readonly outputByteRange: TextRange; readonly outputCodepointRange: TextRange; + readonly outputUtf16Range: TextRange; readonly keyRef?: string; readonly resolvedKeyVersion?: string; readonly tokenRef?: string; @@ -112,7 +118,7 @@ export function init(): Promise; export function scan(text: string, config?: ScanConfig): Finding[]; export function transform( text: string, - findings: Finding[], + findings: FindingInput[], config: TransformationConfig, ): TransformResult; export function scanAndTransform( @@ -123,8 +129,10 @@ export interface PrivacyContext { readonly scope: string; } export interface Restoration { readonly sourceByteRange: TextRange; readonly sourceCodepointRange: TextRange; + readonly sourceUtf16Range: TextRange; readonly outputByteRange: TextRange; readonly outputCodepointRange: TextRange; + readonly outputUtf16Range: TextRange; readonly tokenRef: string; readonly resolvedTokenVersion: string; } diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 393a01c..880e26b 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; -#[derive(Deserialize, Serialize)] +#[derive(Default, Deserialize, Serialize)] struct TextRange { start: usize, end: usize, @@ -23,6 +23,8 @@ struct Finding { matched_text: String, byte_range: TextRange, codepoint_range: TextRange, + #[serde(default, skip_deserializing)] + utf16_range: TextRange, confidence: Option, detector_name: String, detector_version: Option, @@ -54,6 +56,7 @@ struct Transformation { entity_type: String, source_byte_range: TextRange, source_codepoint_range: TextRange, + source_utf16_range: TextRange, confidence: Option, detector_name: String, detector_version: Option, @@ -61,6 +64,7 @@ struct Transformation { replacement: String, output_byte_range: TextRange, output_codepoint_range: TextRange, + output_utf16_range: TextRange, key_ref: Option, resolved_key_version: Option, token_ref: Option, @@ -73,16 +77,23 @@ struct TransformResult { transformations: Vec, } -fn finding_from_core(finding: datafog_core::Finding) -> Finding { - Finding { +fn utf16_range(text: &str, range: datafog_core::TextRange) -> Result { + datafog_core::utf16_range(text, range) + .map(TextRange::from) + .map_err(|error| JsValue::from_str(&error.to_string())) +} + +fn finding_from_core(text: &str, finding: datafog_core::Finding) -> Result { + Ok(Finding { entity_type: finding.entity_type, matched_text: finding.matched_text, byte_range: finding.byte_range.into(), codepoint_range: finding.codepoint_range.into(), + utf16_range: utf16_range(text, finding.byte_range)?, confidence: finding.confidence, detector_name: finding.detector_name, detector_version: finding.detector_version, - } + }) } fn privacy_error(error: datafog_core::PrivacyError) -> JsValue { @@ -102,35 +113,43 @@ fn config_value(config: JsValue) -> Result { serde_wasm_bindgen::from_value(config).map_err(|error| JsValue::from_str(&error.to_string())) } -fn result_to_js(result: datafog_core::TransformResult) -> Result { +fn result_to_js( + source_text: &str, + result: datafog_core::TransformResult, +) -> Result { + let output_text = &result.text; let result = TransformResult { - text: result.text, transformations: result .transformations .into_iter() - .map(|transformation| Transformation { - entity_type: transformation.entity_type, - source_byte_range: transformation.source_byte_range.into(), - source_codepoint_range: transformation.source_codepoint_range.into(), - confidence: transformation.confidence, - detector_name: transformation.detector_name, - detector_version: transformation.detector_version, - strategy: match transformation.strategy { - datafog_core::TransformationStrategy::Redact => "redact", - datafog_core::TransformationStrategy::Remove => "remove", - datafog_core::TransformationStrategy::Mask(_) => "mask", - datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", - datafog_core::TransformationStrategy::Tokenize(_) => "tokenize", - }, - replacement: transformation.replacement, - output_byte_range: transformation.output_byte_range.into(), - output_codepoint_range: transformation.output_codepoint_range.into(), - key_ref: transformation.key_ref, - resolved_key_version: transformation.resolved_key_version, - token_ref: transformation.token_ref, - resolved_token_version: transformation.resolved_token_version, + .map(|transformation| { + Ok(Transformation { + entity_type: transformation.entity_type, + source_byte_range: transformation.source_byte_range.into(), + source_codepoint_range: transformation.source_codepoint_range.into(), + source_utf16_range: utf16_range(source_text, transformation.source_byte_range)?, + confidence: transformation.confidence, + detector_name: transformation.detector_name, + detector_version: transformation.detector_version, + strategy: match transformation.strategy { + datafog_core::TransformationStrategy::Redact => "redact", + datafog_core::TransformationStrategy::Remove => "remove", + datafog_core::TransformationStrategy::Mask(_) => "mask", + datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", + datafog_core::TransformationStrategy::Tokenize(_) => "tokenize", + }, + replacement: transformation.replacement, + output_byte_range: transformation.output_byte_range.into(), + output_codepoint_range: transformation.output_codepoint_range.into(), + output_utf16_range: utf16_range(output_text, transformation.output_byte_range)?, + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, + }) }) - .collect(), + .collect::, JsValue>>()?, + text: result.text, }; serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) @@ -146,8 +165,8 @@ pub fn scan(text: &str, config: Option) -> Result { }; let findings: Vec = datafog_core::scan_with_config(text, &config) .into_iter() - .map(finding_from_core) - .collect(); + .map(|finding| finding_from_core(text, finding)) + .collect::>()?; serde_wasm_bindgen::to_value(&findings).map_err(|error| JsValue::from_str(&error.to_string())) } @@ -186,7 +205,7 @@ pub fn transform(text: &str, findings: JsValue, config: JsValue) -> Result Result) -> std::fmt::Result { + formatter.write_str("UTF-8 byte range is invalid for the supplied text") + } +} + +impl std::error::Error for Utf16RangeError {} + +/// Convert a UTF-8 byte range into zero-based, end-exclusive UTF-16 code-unit +/// offsets for JavaScript consumers. +pub fn utf16_range(text: &str, byte_range: TextRange) -> Result { + if byte_range.start > byte_range.end { + return Err(Utf16RangeError); + } + if byte_range.end > text.len() { + return Err(Utf16RangeError); + } + if !text.is_char_boundary(byte_range.start) || !text.is_char_boundary(byte_range.end) { + return Err(Utf16RangeError); + } + + Ok(TextRange { + start: text[..byte_range.start].encode_utf16().count(), + end: text[..byte_range.end].encode_utf16().count(), + }) +} + /// A piece of potentially sensitive content detected in an input string. #[derive(Debug, Clone, PartialEq)] pub struct Finding { @@ -3432,6 +3463,7 @@ mod tests { TokenProviderErrorKind, TokenizeProviderFuture, TokenizeResult, TransformationConfig, TransformationStrategy, parse_scan_and_transform_config, parse_transformation_config, required_restore_items, restore_with_results, scan, scan_and_transform, transform, + utf16_range, }; use futures::executor::block_on; use serde_json::json; @@ -3439,6 +3471,21 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; + #[test] + fn utf16_ranges_match_javascript_string_offsets() { + let text = "👋 jane@example.com"; + + assert_eq!( + utf16_range(text, TextRange { start: 5, end: 21 }), + Ok(TextRange { start: 3, end: 19 }) + ); + } + + #[test] + fn utf16_ranges_reject_invalid_byte_boundaries() { + assert!(utf16_range("👋", TextRange { start: 1, end: 4 }).is_err()); + } + #[derive(Default)] struct TestKeyProvider { responses: BTreeMap<(String, Option), (Vec, String)>, diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index 009b9f7..d207b60 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -266,7 +266,11 @@ All ranges are zero-based and end-exclusive. - The Rust implementation uses UTF-8 byte ranges internally. - The canonical public default is Unicode code-point ranges. - Every public range states its coordinate unit explicitly. -- UTF-16 code-unit ranges are supported for JavaScript consumers. +- Node.js and browser WASM findings always include `utf16Range`. Their + transformation records always include `sourceUtf16Range` and + `outputUtf16Range`; Node restoration records use the same source/output + names. These ranges select the same spans with JavaScript string operations. + Caller-supplied findings do not require the derived `utf16Range` field. - Findings expose both UTF-8 byte and Unicode code-point ranges. Bindings may derive additional native ranges without silently changing the meaning of a field. @@ -366,6 +370,8 @@ end-exclusive, refer to the transformed text, and select exactly `replacement`. Source metadata deliberately excludes `matched_text`; Core does not echo the original PII or offer an include-originals switch. Callers that explicitly need the source value already possess the input and can use the source ranges. +JavaScript binding records add the UTF-16 source/output fields defined under +Text ranges without changing the byte or code-point fields. Pseudonymization records include the configured key reference and the concrete version returned by the provider. They never include key material or a @@ -531,7 +537,8 @@ database, vault, or cryptographic adapter. record carries token source and restored output byte/code-point ranges, `token_ref`, and the concrete token profile version. It does not duplicate restored plaintext or expose payload, scope, credentials, provider topology, -or a token-to-plaintext mapping. +or a token-to-plaintext mapping. Node restoration records additionally carry +`sourceUtf16Range` and `outputUtf16Range`. ## Capability-continuity stance diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md index 0779998..e24e21f 100644 --- a/docs/privacy-capability-matrix.md +++ b/docs/privacy-capability-matrix.md @@ -6,7 +6,7 @@ for a PII detection and transformation engine. | Capability | Treatment | Core direction | | --- | --- | --- | -| PII scanning | Preserve | Return validated findings with explicit byte and code-point ranges, optional confidence, and detector provenance. | +| PII scanning | Preserve | Return validated findings with explicit byte and code-point ranges, optional confidence, and detector provenance; JavaScript bindings also expose explicit UTF-16 ranges. | | Typed redaction | Preserve | Replace findings with typed, document-local placeholders after deterministic overlap resolution. | | Character masking | Preserve | Mask Unicode code points with a validated character and explicit leading- or trailing-reveal semantics. | | Entity-type selection | Preserve | Select canonical entity types through transformation configuration. | diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 44e8e5c..d47ffd6 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -207,21 +207,24 @@ while browser WASM rejects provider-backed work. ## Slice 8: Binding completion and release hardening +**Status: complete.** + Rust, Python, and Node implement all capabilities through Slice 7. Browser WASM implements the stateless transformations through Slice 4 and explicitly rejects provider-backed pseudonymization. New stateless operations should continue to ship through the bindings in the same vertical slice as their Rust implementation rather than waiting for a separate binding rollout. -Remaining binding work is: +Node.js and browser WASM findings expose `utf16Range`; their transformation +records expose `sourceUtf16Range` and `outputUtf16Range`; Node restoration +records expose the same source/output names. All are zero-based, end-exclusive +UTF-16 code-unit ranges. One validated Core helper derives them from canonical +byte ranges, while existing byte and code-point fields remain unchanged. -1. add explicitly named UTF-16 code-unit ranges for JavaScript consumers as - required by ADR 001, without changing the existing byte or code-point - fields; -2. retain Rust, Python, and Node provider-backed conformance coverage while - keeping browser WASM key and token providers explicitly unsupported; and -3. retain installed-package and cross-binding conformance tests as release - gates. +Installed-package tests prove emoji-prefixed findings and transformation or +restoration records select the exact spans with JavaScript `slice`. Existing +Rust, Python, and Node provider-backed conformance coverage remains in place, +and browser WASM continues to reject key- and token-provider work explicitly. Pseudonymization and reversible token storage are not promised in browser WASM without a separately accepted host-managed key-custody boundary. diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index 916a353..b883431 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -57,6 +57,10 @@ function verifyContract(text, finding) { .join(""), finding.matchedText, ); + assert.equal( + text.slice(finding.utf16Range.start, finding.utf16Range.end), + finding.matchedText, + ); assert.equal(finding.confidence, undefined); assert.ok(finding.detectorName.startsWith("datafog-core/")); assert.ok(finding.detectorVersion); @@ -82,6 +86,14 @@ for (const name of ["development.jsonl", "final.jsonl"]) { const emojiFinding = scan("👋 jane@example.com")[0]; assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); +assert.deepEqual(emojiFinding.utf16Range, { start: 3, end: 19 }); +const { utf16Range: _derivedRange, ...preSliceEightFinding } = emojiFinding; +assert.equal( + transform("👋 jane@example.com", [preSliceEightFinding], { + default: { strategy: "redact" }, + }).text, + "👋 [EMAIL]", +); assert.deepEqual( scan("Email jane@example.com", { locale: "en-US" }), scan("Email jane@example.com"), @@ -100,6 +112,18 @@ assert.equal(explicit.transformations.length, 2); assert.equal(explicit.transformations[0].replacement, "[EMAIL]"); assert.deepEqual(explicit.transformations[0].outputByteRange, { start: 5, end: 12 }); assert.deepEqual(explicit.transformations[0].outputCodepointRange, { start: 2, end: 9 }); +assert.deepEqual(explicit.transformations[0].sourceUtf16Range, { start: 3, end: 19 }); +assert.deepEqual(explicit.transformations[0].outputUtf16Range, { start: 3, end: 10 }); +for (const record of explicit.transformations) { + assert.equal( + transformText.slice(record.sourceUtf16Range.start, record.sourceUtf16Range.end), + "jane@example.com", + ); + assert.equal( + explicit.text.slice(record.outputUtf16Range.start, record.outputUtf16Range.end), + record.replacement, + ); +} assert.equal("finding" in explicit.transformations[0], false); assert.equal("matchedText" in explicit.transformations[0], false); assert.equal(explicit.transformations[0].entityType, "EMAIL"); @@ -283,6 +307,17 @@ assert.equal(tokenized.transformations[0].resolvedTokenVersion, "active-1"); const restored = await tokenManager.restore(tokenized.text, tokenContext); assert.equal(restored.text, "👋 jane@example.com jane@example.com"); assert.equal(restored.restorations.length, 2); +for (const record of restored.restorations) { + assert.ok( + tokenized.text + .slice(record.sourceUtf16Range.start, record.sourceUtf16Range.end) + .startsWith("DFTOKENv1("), + ); + assert.equal( + restored.text.slice(record.outputUtf16Range.start, record.outputUtf16Range.end), + "jane@example.com", + ); +} await assert.rejects( tokenManager.restore(tokenized.text, { scope: "tenant/b" }), (error) => error instanceof DataFogError && error.code === "token_access_denied", @@ -317,6 +352,7 @@ import { PrivacyManager, type EntityType, type Finding, + type FindingInput, type MaskRevealConfig, type KeyProvider, type ScanAndTransformConfig, @@ -326,8 +362,10 @@ import { } from "@datafog/node"; const findings: Finding[] = scan("Email jane@example.com"); +const suppliedFinding: FindingInput = findings[0]; const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; +const utf16Range: TextRange = findings[0]?.utf16Range ?? { start: 0, end: 0 }; const explicit: TransformResult = transform( "Email jane@example.com", findings, @@ -359,7 +397,9 @@ const pseudonymized: Promise = new PrivacyManager(provider).tra ); void entityType; +void suppliedFinding; void range; +void utf16Range; void explicit; void convenience; void masked; diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index 8641048..c6fbba1 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -86,6 +86,7 @@ import { transform, type EntityType, type Finding, + type FindingInput, type MaskRevealConfig, type ScanAndTransformConfig, type TextRange, @@ -96,8 +97,10 @@ import { const ready: Promise = init(); const findings: Finding[] = scan("Email jane@example.com"); +const suppliedFinding: FindingInput = findings[0]; const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; +const utf16Range: TextRange = findings[0]?.utf16Range ?? { start: 0, end: 0 }; const transformed: TransformResult = transform( "Email jane@example.com", findings, @@ -121,7 +124,9 @@ const unchanged: RestoreResult = restore("ordinary text", { scope: "tenant" }); void ready; void entityType; +void suppliedFinding; void range; +void utf16Range; void transformed; void scannedAndTransformed; void masked; @@ -242,6 +247,12 @@ try { if (matchedCodepoints !== finding.matchedText) { throw new Error("code-point range does not select matched text"); } + if ( + text.slice(finding.utf16Range.start, finding.utf16Range.end) !== + finding.matchedText + ) { + throw new Error("UTF-16 range does not select matched text"); + } if (finding.confidence !== undefined) { throw new Error("rule-based findings must omit confidence"); } @@ -272,10 +283,23 @@ try { const emojiFinding = scan("👋 jane@example.com")[0]; if ( JSON.stringify(emojiFinding.byteRange) !== JSON.stringify({ start: 5, end: 21 }) || - JSON.stringify(emojiFinding.codepointRange) !== JSON.stringify({ start: 2, end: 18 }) + JSON.stringify(emojiFinding.codepointRange) !== JSON.stringify({ start: 2, end: 18 }) || + JSON.stringify(emojiFinding.utf16Range) !== JSON.stringify({ start: 3, end: 19 }) || + "👋 jane@example.com".slice( + emojiFinding.utf16Range.start, + emojiFinding.utf16Range.end, + ) !== emojiFinding.matchedText ) { throw new Error("Unicode ranges do not use the documented coordinate systems"); } + const { utf16Range: _derivedRange, ...preSliceEightFinding } = emojiFinding; + if ( + transform("👋 jane@example.com", [preSliceEightFinding], { + default: { strategy: "redact" }, + }).text !== "👋 [EMAIL]" + ) { + throw new Error("UTF-16 output fields changed the accepted finding input shape"); + } if ( JSON.stringify(scan("Email jane@example.com", { locale: "en-US" })) !== JSON.stringify(scan("Email jane@example.com")) @@ -303,6 +327,10 @@ try { (record) => record.strategy !== "redact" || record.replacement !== "[EMAIL]" || + text.slice(record.sourceUtf16Range.start, record.sourceUtf16Range.end) !== + "jane@example.com" || + explicit.text.slice(record.outputUtf16Range.start, record.outputUtf16Range.end) !== + record.replacement || Array.from(explicit.text) .slice(record.outputCodepointRange.start, record.outputCodepointRange.end) .join("") !== record.replacement,