diff --git a/README.md b/README.md index 4221456..15bcb5e 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,16 @@ Fast structured PII detection, implemented in Rust and exposed for Rust, Python, Node.js, and browsers. -It detects `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, and `ZIP_CODE`. Every binding returns the same entity shape: +It detects `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, and `ZIP_CODE`. Every binding returns the same finding information: ```text -label, text, start, end +entity type, matched text, byte range, code-point range, +optional confidence, detector name, optional detector version ``` -`start` and `end` are zero-based Unicode code-point offsets; `end` is exclusive. +Both ranges use zero-based, end-exclusive offsets. The byte range addresses the +UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based +detectors currently report no confidence score. ## Packages @@ -30,9 +33,10 @@ cargo add datafog-core ```rust use datafog_core::scan; -let entities = scan("Email jane@example.com"); -assert_eq!(entities[0].label, "EMAIL"); -assert_eq!(entities[0].text, "jane@example.com"); +let findings = scan("Email jane@example.com"); +assert_eq!(findings[0].entity_type, "EMAIL"); +assert_eq!(findings[0].matched_text, "jane@example.com"); +assert_eq!(findings[0].byte_range.start, 6); ``` ### Python @@ -44,9 +48,10 @@ python -m pip install datafog-core ```python from datafog_core import scan -entities = scan("Email jane@example.com") -print(entities[0].label) # EMAIL -print(entities[0].text) # jane@example.com +findings = scan("Email jane@example.com") +print(findings[0].entity_type) # EMAIL +print(findings[0].matched_text) # jane@example.com +print(findings[0].byte_range.start) # 6 ``` ### Node.js diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index 5531143..90bc453 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -1,8 +1,2 @@ -export type Label = - | "EMAIL" - | "PHONE" - | "SSN" - | "CREDIT_CARD" - | "IP_ADDRESS" - | "DATE" - | "ZIP_CODE"; +/** Canonical built-in values are uppercase, but custom detectors may add values. */ +export type EntityType = string; diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index a5a612b..325979f 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -1,17 +1,19 @@ -export type Label = - | "EMAIL" - | "PHONE" - | "SSN" - | "CREDIT_CARD" - | "IP_ADDRESS" - | "DATE" - | "ZIP_CODE"; -export interface Entity { - readonly label: Label - readonly text: string +/** Canonical built-in values are uppercase, but custom detectors may add values. */ +export type EntityType = string; +export interface Finding { + readonly entityType: EntityType + readonly matchedText: string + readonly byteRange: TextRange + readonly codepointRange: TextRange + readonly confidence?: number + readonly detectorName: string + readonly detectorVersion?: string +} + +/** Scan text for supported PII findings. */ +export declare function scan(text: string): Array + +export interface TextRange { readonly start: number readonly end: number } - -/** Scan text for supported PII entities. */ -export declare function scan(text: string): Array diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index d540e3d..3b4733f 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -4,18 +4,36 @@ use napi::{Error, Status}; use napi_derive::napi; #[napi(object, object_from_js = false)] -pub struct Entity { - #[napi(readonly, ts_type = "Label")] - pub label: String, +pub struct TextRange { + #[napi(readonly)] + pub start: u32, + + #[napi(readonly)] + pub end: u32, +} + +#[napi(object, object_from_js = false)] +pub struct Finding { + #[napi(readonly, ts_type = "EntityType")] + pub entity_type: String, #[napi(readonly)] - pub text: String, + pub matched_text: String, #[napi(readonly)] - pub start: u32, + pub byte_range: TextRange, #[napi(readonly)] - pub end: u32, + pub codepoint_range: TextRange, + + #[napi(readonly)] + pub confidence: Option, + + #[napi(readonly)] + pub detector_name: String, + + #[napi(readonly)] + pub detector_version: Option, } fn js_offset(offset: usize) -> napi::Result { @@ -27,17 +45,27 @@ fn js_offset(offset: usize) -> napi::Result { }) } -/// Scan text for supported PII entities. +fn js_range(range: datafog_core::TextRange) -> napi::Result { + Ok(TextRange { + start: js_offset(range.start)?, + end: js_offset(range.end)?, + }) +} + +/// Scan text for supported PII findings. #[napi(strict, catch_unwind)] -pub fn scan(text: String) -> napi::Result> { +pub fn scan(text: String) -> napi::Result> { datafog_core::scan(&text) .into_iter() - .map(|entity| { - Ok(Entity { - label: entity.label, - text: entity.text, - start: js_offset(entity.start)?, - end: js_offset(entity.end)?, + .map(|finding| { + Ok(Finding { + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: js_range(finding.byte_range)?, + codepoint_range: js_range(finding.codepoint_range)?, + confidence: finding.confidence.map(f64::from), + detector_name: finding.detector_name, + detector_version: finding.detector_version, }) }) .collect() diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 0405122..21d360a 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -2,61 +2,116 @@ use ::datafog_core as core; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -/// An entity detected in input text. +/// A zero-based, end-exclusive text range. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone, PartialEq, Eq)] +struct TextRange { + #[pyo3(get)] + start: usize, + + #[pyo3(get)] + end: usize, +} + +impl From for TextRange { + fn from(range: core::TextRange) -> Self { + Self { + start: range.start, + end: range.end, + } + } +} + +#[pymethods] +impl TextRange { + fn __repr__(&self) -> String { + format!("TextRange(start={}, end={})", self.start, self.end) + } + + fn __eq__(&self, other: PyRef<'_, TextRange>) -> bool { + self == &*other + } +} + +/// A piece of potentially sensitive content detected in input text. #[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] -struct Entity { +struct Finding { #[pyo3(get)] - label: String, + entity_type: String, #[pyo3(get)] - text: String, + matched_text: String, #[pyo3(get)] - start: usize, + byte_range: TextRange, #[pyo3(get)] - end: usize, + codepoint_range: TextRange, + + #[pyo3(get)] + confidence: Option, + + #[pyo3(get)] + detector_name: String, + + #[pyo3(get)] + detector_version: Option, } -impl From for Entity { - fn from(entity: core::Entity) -> Self { +impl From for Finding { + fn from(finding: core::Finding) -> Self { Self { - label: entity.label, - text: entity.text, - start: entity.start, - end: entity.end, + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: finding.byte_range.into(), + codepoint_range: finding.codepoint_range.into(), + confidence: finding.confidence, + detector_name: finding.detector_name, + detector_version: finding.detector_version, } } } #[pymethods] -impl Entity { +impl Finding { fn __repr__(&self) -> String { format!( - "Entity(label={:?}, text={:?}, start={}, end={})", - self.label, self.text, self.start, self.end + "Finding(entity_type={:?}, matched_text={:?}, byte_range={:?}, \ + codepoint_range={:?}, confidence={:?}, detector_name={:?}, \ + detector_version={:?})", + self.entity_type, + self.matched_text, + (self.byte_range.start, self.byte_range.end), + (self.codepoint_range.start, self.codepoint_range.end), + self.confidence, + self.detector_name, + self.detector_version, ) } - fn __eq__(&self, other: PyRef<'_, Entity>) -> bool { - self.label == other.label - && self.text == other.text - && self.start == other.start - && self.end == other.end + fn __eq__(&self, other: PyRef<'_, Finding>) -> bool { + self.entity_type == other.entity_type + && self.matched_text == other.matched_text + && self.byte_range == other.byte_range + && self.codepoint_range == other.codepoint_range + && self.confidence == other.confidence + && self.detector_name == other.detector_name + && self.detector_version == other.detector_version } } -/// Scan text for supported PII entities. +/// Scan text for supported PII findings. #[pyfunction] -fn scan(text: &str) -> PyResult> { - std::panic::catch_unwind(|| core::scan(text).into_iter().map(Entity::from).collect()) +fn scan(text: &str) -> PyResult> { + std::panic::catch_unwind(|| core::scan(text).into_iter().map(Finding::from).collect()) .map_err(|_| PyRuntimeError::new_err("unexpected Rust scan failure")) } #[pymodule] fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(scan, module)?)?; Ok(()) } diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index 9b0f17c..5178468 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -17,11 +17,32 @@ def expected_entities(record: dict[str, object]) -> list[dict[str, object]]: def actual_entities(text: str) -> list[dict[str, object]]: return [ - {"label": entity.label, "text": entity.text, "start": entity.start, "end": entity.end} - for entity in scan(text) + { + "label": finding.entity_type, + "text": finding.matched_text, + "start": finding.codepoint_range.start, + "end": finding.codepoint_range.end, + } + for finding in scan(text) ] +def verify_contract(text: str) -> None: + encoded = text.encode("utf-8") + for finding in scan(text): + assert ( + encoded[finding.byte_range.start : finding.byte_range.end].decode("utf-8") + == finding.matched_text + ) + assert ( + text[finding.codepoint_range.start : finding.codepoint_range.end] + == finding.matched_text + ) + assert finding.confidence is None + assert finding.detector_name.startswith("datafog-core/") + assert finding.detector_version + + def verify_fixture(name: str) -> None: path = ROOT / "fixtures" / name for line in path.read_text().splitlines(): @@ -29,12 +50,16 @@ def verify_fixture(name: str) -> None: actual = actual_entities(record["text"]) expected = expected_entities(record) assert actual == expected, record["id"] + verify_contract(record["text"]) def main() -> None: verify_fixture("development.jsonl") verify_fixture("final.jsonl") - print("Installed datafog_core wheel matches both fixtures.") + emoji_finding = scan("๐Ÿ‘‹ jane@example.com")[0] + assert (emoji_finding.byte_range.start, emoji_finding.byte_range.end) == (5, 21) + assert (emoji_finding.codepoint_range.start, emoji_finding.codepoint_range.end) == (2, 18) + print("Installed datafog_core wheel matches fixtures and the Finding contract.") if __name__ == "__main__": diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index 957d44d..dfe6171 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -1,18 +1,20 @@ -export type Label = - | "EMAIL" - | "PHONE" - | "SSN" - | "CREDIT_CARD" - | "IP_ADDRESS" - | "DATE" - | "ZIP_CODE"; +/** Canonical built-in values are uppercase, but custom detectors may add values. */ +export type EntityType = string; -export interface Entity { - readonly label: Label; - readonly text: string; +export interface TextRange { readonly start: number; readonly end: number; } +export interface Finding { + readonly entityType: EntityType; + readonly matchedText: string; + readonly byteRange: TextRange; + readonly codepointRange: TextRange; + readonly confidence?: number; + readonly detectorName: string; + readonly detectorVersion?: string; +} + export function init(): Promise; -export function scan(text: string): Entity[]; +export function scan(text: string): Finding[]; diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 52d6083..304184e 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -2,24 +2,46 @@ use serde::Serialize; use wasm_bindgen::prelude::*; #[derive(Serialize)] -struct Entity { - label: String, - text: String, +struct TextRange { start: usize, end: usize, } +impl From for TextRange { + fn from(range: datafog_core::TextRange) -> Self { + Self { + start: range.start, + end: range.end, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Finding { + entity_type: String, + matched_text: String, + byte_range: TextRange, + codepoint_range: TextRange, + confidence: Option, + detector_name: String, + detector_version: Option, +} + #[wasm_bindgen] pub fn scan(text: &str) -> Result { - let entities: Vec = datafog_core::scan(text) + let findings: Vec = datafog_core::scan(text) .into_iter() - .map(|entity| Entity { - label: entity.label, - text: entity.text, - start: entity.start, - end: entity.end, + .map(|finding| Finding { + entity_type: finding.entity_type, + matched_text: finding.matched_text, + byte_range: finding.byte_range.into(), + codepoint_range: finding.codepoint_range.into(), + confidence: finding.confidence, + detector_name: finding.detector_name, + detector_version: finding.detector_version, }) .collect(); - serde_wasm_bindgen::to_value(&entities).map_err(|error| JsValue::from_str(&error.to_string())) + serde_wasm_bindgen::to_value(&findings).map_err(|error| JsValue::from_str(&error.to_string())) } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index a285222..19b4905 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,22 +1,37 @@ -//! Core PII scanning API for the DataFog Rust proof of concept. +//! Core PII scanning API for DataFog. use regex::Regex; use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::LazyLock; -/// A PII entity detected in an input string. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Entity { - /// The detected PII label, such as `EMAIL` or `SSN`. - pub label: String, - /// The exact matched substring from the input text. - pub text: String, - /// Zero-based Unicode code-point offset where the entity starts. +/// A zero-based, end-exclusive range in the coordinate system named by its +/// containing field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TextRange { + /// Inclusive start offset. pub start: usize, - /// Exclusive Unicode code-point offset where the entity ends. + /// Exclusive end offset. pub end: usize, } -/// private enum +/// A piece of potentially sensitive content detected in an input string. +#[derive(Debug, Clone, PartialEq)] +pub struct Finding { + /// Canonical PII type, such as `EMAIL` or `SSN`. + pub entity_type: String, + /// Exact substring matched in the original input. + pub matched_text: String, + /// Range in UTF-8 bytes. + pub byte_range: TextRange, + /// Range in Unicode code points. + pub codepoint_range: TextRange, + /// Detection confidence in `0.0..=1.0`, when the detector produces one. + pub confidence: Option, + /// Stable name of the detector that produced this finding. + pub detector_name: String, + /// Version of the detector implementation, when available. + pub detector_version: Option, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] enum Label { Email, @@ -41,6 +56,18 @@ impl Label { Label::ZipCode => "ZIP_CODE", } } + + fn detector_name(self) -> &'static str { + match self { + Label::Email => "datafog-core/email", + Label::Phone => "datafog-core/phone", + Label::Ssn => "datafog-core/ssn", + Label::CreditCard => "datafog-core/credit-card", + Label::IpAddress => "datafog-core/ip-address", + Label::Date => "datafog-core/date", + Label::ZipCode => "datafog-core/zip-code", + } + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -50,9 +77,8 @@ struct Candidate { end_byte: usize, } -/// Scan text for supported PII entities. -/// -pub fn scan(text: &str) -> Vec { +/// Scan text for supported PII findings. +pub fn scan(text: &str) -> Vec { let mut candidates: Vec = Vec::new(); detect_email(text, &mut candidates); detect_phone(text, &mut candidates); @@ -64,7 +90,7 @@ pub fn scan(text: &str) -> Vec { finalize(text, candidates) } -fn finalize(text: &str, mut candidates: Vec) -> Vec { +fn finalize(text: &str, mut candidates: Vec) -> Vec { candidates.sort_by(|left, right| { left.start_byte .cmp(&right.start_byte) @@ -98,11 +124,17 @@ fn finalize(text: &str, mut candidates: Vec) -> Vec { code_point_offset(text, candidate.end_byte) }; - Entity { - label: candidate.label.as_str().to_owned(), - text: text[candidate.start_byte..candidate.end_byte].to_owned(), - start, - end, + Finding { + entity_type: candidate.label.as_str().to_owned(), + matched_text: text[candidate.start_byte..candidate.end_byte].to_owned(), + byte_range: TextRange { + start: candidate.start_byte, + end: candidate.end_byte, + }, + codepoint_range: TextRange { start, end }, + confidence: None, + detector_name: candidate.label.detector_name().to_owned(), + detector_version: Some(env!("CARGO_PKG_VERSION").to_owned()), } }) .collect() @@ -703,7 +735,47 @@ fn detect_ip_address(text: &str, candidates: &mut Vec) { #[cfg(test)] mod tests { - use super::{Entity, scan}; + use super::{Finding, TextRange, scan}; + + fn expected_finding( + entity_type: &str, + matched_text: &str, + byte_range: (usize, usize), + codepoint_range: (usize, usize), + detector_name: &str, + ) -> Finding { + Finding { + entity_type: entity_type.to_owned(), + matched_text: matched_text.to_owned(), + byte_range: TextRange { + start: byte_range.0, + end: byte_range.1, + }, + codepoint_range: TextRange { + start: codepoint_range.0, + end: codepoint_range.1, + }, + confidence: None, + detector_name: detector_name.to_owned(), + detector_version: Some(env!("CARGO_PKG_VERSION").to_owned()), + } + } + + fn expected_ascii_finding( + entity_type: &str, + matched_text: &str, + start: usize, + end: usize, + detector_name: &str, + ) -> Finding { + expected_finding( + entity_type, + matched_text, + (start, end), + (start, end), + detector_name, + ) + } #[test] fn empty_input_has_no_entities() { @@ -714,25 +786,41 @@ mod tests { fn detects_email_without_trailing_punctuation() { assert_eq!( scan("Contact jane@example.com."), - vec![Entity { - label: "EMAIL".to_owned(), - text: "jane@example.com".to_owned(), - start: 8, - end: 24, - }] + vec![expected_ascii_finding( + "EMAIL", + "jane@example.com", + 8, + 24, + "datafog-core/email", + )] ); } #[test] - fn reports_email_offsets_as_unicode_code_points() { + fn reports_email_byte_and_codepoint_ranges() { + let text = "๐Ÿ‘‹ jane@example.com"; + assert_eq!( + scan(text), + vec![expected_finding( + "EMAIL", + "jane@example.com", + (5, 21), + (2, 18), + "datafog-core/email", + )] + ); + + let finding = &scan(text)[0]; + assert_eq!( + &text[finding.byte_range.start..finding.byte_range.end], + finding.matched_text + ); assert_eq!( - scan("๐Ÿ‘‹ jane@example.com"), - vec![Entity { - label: "EMAIL".to_owned(), - text: "jane@example.com".to_owned(), - start: 2, - end: 18, - }] + text.chars() + .skip(finding.codepoint_range.start) + .take(finding.codepoint_range.end - finding.codepoint_range.start) + .collect::(), + finding.matched_text ); } @@ -740,12 +828,13 @@ mod tests { fn detects_north_american_phone_number() { assert_eq!( scan("Call (212) 555-0100."), - vec![Entity { - label: "PHONE".to_owned(), - text: "(212) 555-0100".to_owned(), - start: 5, - end: 19, - }] + vec![expected_ascii_finding( + "PHONE", + "(212) 555-0100", + 5, + 19, + "datafog-core/phone", + )] ); } @@ -753,12 +842,13 @@ mod tests { fn detects_international_phone_number() { assert_eq!( scan("Intl +44 20 7946 0958"), - vec![Entity { - label: "PHONE".to_owned(), - text: "+44 20 7946 0958".to_owned(), - start: 5, - end: 21, - }] + vec![expected_ascii_finding( + "PHONE", + "+44 20 7946 0958", + 5, + 21, + "datafog-core/phone", + )] ); } @@ -772,18 +862,19 @@ mod tests { fn detects_dashed_ssn() { assert_eq!( scan("SSN: 123-45-6789"), - vec![Entity { - label: "SSN".to_owned(), - text: "123-45-6789".to_owned(), - start: 5, - end: 16, - }] + vec![expected_ascii_finding( + "SSN", + "123-45-6789", + 5, + 16, + "datafog-core/ssn", + )] ); } #[test] fn detects_undashed_ssn() { - assert_eq!(scan("123456789")[0].label, "SSN"); + assert_eq!(scan("123456789")[0].entity_type, "SSN"); } #[test] @@ -800,20 +891,21 @@ mod tests { fn detects_formatted_credit_card() { assert_eq!( scan("Card: 4111-1111-1111-1111"), - vec![Entity { - label: "CREDIT_CARD".to_owned(), - text: "4111-1111-1111-1111".to_owned(), - start: 6, - end: 25, - }] + vec![expected_ascii_finding( + "CREDIT_CARD", + "4111-1111-1111-1111", + 6, + 25, + "datafog-core/credit-card", + )] ); } #[test] fn detects_supported_card_issuers() { - assert_eq!(scan("5555 5555 5555 4444")[0].label, "CREDIT_CARD"); - assert_eq!(scan("378282246310005")[0].label, "CREDIT_CARD"); - assert_eq!(scan("6011111111111117")[0].label, "CREDIT_CARD"); + assert_eq!(scan("5555 5555 5555 4444")[0].entity_type, "CREDIT_CARD"); + assert_eq!(scan("378282246310005")[0].entity_type, "CREDIT_CARD"); + assert_eq!(scan("6011111111111117")[0].entity_type, "CREDIT_CARD"); } #[test] @@ -827,16 +919,17 @@ mod tests { fn detects_numeric_and_named_dates() { assert_eq!( scan("Date: 2024-02-29"), - vec![Entity { - label: "DATE".to_owned(), - text: "2024-02-29".to_owned(), - start: 6, - end: 16, - }] + vec![expected_ascii_finding( + "DATE", + "2024-02-29", + 6, + 16, + "datafog-core/date", + )] ); - assert_eq!(scan("12/27/1988")[0].label, "DATE"); - assert_eq!(scan("Jan 15, 2024")[0].label, "DATE"); + assert_eq!(scan("12/27/1988")[0].entity_type, "DATE"); + assert_eq!(scan("Jan 15, 2024")[0].entity_type, "DATE"); } #[test] @@ -852,15 +945,16 @@ mod tests { fn detects_five_digit_and_plus_four_zip_codes() { assert_eq!( scan("ZIP: 94105-1234"), - vec![Entity { - label: "ZIP_CODE".to_owned(), - text: "94105-1234".to_owned(), - start: 5, - end: 15, - }] + vec![expected_ascii_finding( + "ZIP_CODE", + "94105-1234", + 5, + 15, + "datafog-core/zip-code", + )] ); - assert_eq!(scan("94105")[0].label, "ZIP_CODE"); + assert_eq!(scan("94105")[0].entity_type, "ZIP_CODE"); } #[test] @@ -876,17 +970,18 @@ mod tests { fn detects_ipv4_and_ipv6_addresses() { assert_eq!( scan("IPv4: 192.168.1.10"), - vec![Entity { - label: "IP_ADDRESS".to_owned(), - text: "192.168.1.10".to_owned(), - start: 6, - end: 18, - }] + vec![expected_ascii_finding( + "IP_ADDRESS", + "192.168.1.10", + 6, + 18, + "datafog-core/ip-address", + )] ); - assert_eq!(scan("2001:db8::1")[0].label, "IP_ADDRESS"); - assert_eq!(scan("[2001:db8::1]:443")[0].text, "2001:db8::1"); - assert_eq!(scan("192.168.1.10:8080")[0].text, "192.168.1.10"); + assert_eq!(scan("2001:db8::1")[0].entity_type, "IP_ADDRESS"); + assert_eq!(scan("[2001:db8::1]:443")[0].matched_text, "2001:db8::1"); + assert_eq!(scan("192.168.1.10:8080")[0].matched_text, "192.168.1.10"); } #[test] diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md new file mode 100644 index 0000000..974ecf0 --- /dev/null +++ b/docs/adr/001-privacy-core-contract.md @@ -0,0 +1,180 @@ +# ADR 001: Privacy Core Contract + +- **Status:** Accepted +- **Date:** 2026-08-27 + +## Context + +DataFog Core is a PII detection and transformation engine. + +The engine is organized around three operations: + +```text +scan(text) -> findings +transform(...) -> transformed text and transformation records +restore(...) -> authorized restoration of reversible tokens +``` + +## Decision + +### Operation taxonomy + +Canonical core operations are `scan`, `transform`, and `restore`. + +Canonical transformation strategies are: + +```text +remove +redact +mask +hash +pseudonymize +tokenize +``` + +`allow`, `warn`, and `block` are governance decisions, not Core operations or +transformation strategies. A separate governance layer may consume findings +and transformation results to make those decisions. The calling application, +proxy, or firewall is responsible for enforcing them. + +### Text ranges + +All ranges are zero-based and end-exclusive. + +- The Rust implementation uses UTF-8 byte ranges internally. +- The canonical public default is Unicode code-point ranges. +- Every public range states its coordinate unit explicitly. +- UTF-16 code-unit ranges are supported for JavaScript consumers. +- Findings expose both UTF-8 byte and Unicode code-point ranges. Bindings may + derive additional native ranges without silently changing the meaning of a + field. +- Ranges always refer to the exact original input. The engine performs no + implicit Unicode normalization. + +### Finding contract + +A finding contains: + +```text +Finding { + entity_type + matched_text + byte_range + codepoint_range + confidence? + detector_name + detector_version? +} +``` + +Built-in entity types use canonical uppercase names. The public entity-type +field remains extensible rather than being restricted to a closed enum. +Confidence, when present, is in the inclusive range `0.0..=1.0`. + +### Supplied-finding validation + +Transformations reject the entire request when any externally supplied finding +is malformed. Invalid findings are not silently ignored. Invalid conditions +include: + +- an empty, reversed, or out-of-bounds range; +- inconsistent byte and code-point ranges; +- a range that does not fall on a valid boundary; +- `matched_text` that differs from the referenced source substring; or +- confidence outside `0.0..=1.0`. + +Findings produced by the core scanner must satisfy these invariants by +construction. + +### Duplicates and overlaps + +`scan` may report meaningful overlaps. `transform` resolves them before making +changes. + +Exact duplicates have the same entity type, source range, and matched text and +collapse into one finding. If duplicate detector results differ, the result +with higher confidence is retained; remaining ties use stable detector-name +ordering. + +Overlapping findings are ranked by: + +1. longer source span; +2. higher entity-type priority; +3. higher confidence; +4. earlier source position; and +5. stable lexical ordering. + +The selected non-overlapping findings are returned in document order. Entity +priority has a core default and may later be made configurable. + +### Transformation result + +The canonical result follows the transformed-text-plus-entity-record pattern +used by established PII tools: + +```text +TransformResult { + text + transformations: [Transformation] +} + +Transformation { + finding + strategy + replacement + output_range +} +``` + +Transformation records are ordered by source document position and include +only transformations that were actually applied. The finding already supplies +the original value and source range, so the canonical payload does not contain +a second original-to-replacement mapping. + +A mapping view may be offered as an explicit convenience API. Sensitive +original values are excluded from default debug and log output. + +### Security meaning of strategies + +- `redact` creates typed, document-local placeholders. +- `mask` hides all or a configured part of a value. +- `hash` is a compatibility fingerprint with explicitly documented leakage and + must not be presented as secure pseudonymization. +- `pseudonymize` is a new keyed, scoped, deterministic, one-way operation. It + is not based on the Python numbered-placeholder behavior. +- `tokenize` creates opaque reversible or vault-backed tokens. +- `restore` accepts only explicitly reversible tokens and requires + authorization. + +Pseudonymization is value pseudonymization, not identity resolution. The core +does not infer that different identifiers belong to the same person. + +## Capability-continuity stance + +Python behavior is classified as preserved, redesigned, compatibility-only, or +dropped. Compatibility means preserving valuable user capabilities, not +reproducing every Python output. Exact-output compatibility must be justified +and documented separately. + +## Consequences + +- Bindings cannot treat unlabeled `start` and `end` fields as universally + portable. +- Strict finding validation may intentionally differ from permissive legacy + behavior. +- Consumers receive auditable transformation records without a redundant + mapping dictionary. +- Secure pseudonymization and tokenization require explicit key and scope + contracts in later ADRs. +- Pseudonymized data remains sensitive and must not be described as anonymous. +- Governance decisions and payload enforcement remain outside DataFog Core. + +## Deferred decisions + +This ADR does not choose: + +- mask length and direction options; +- compatibility hash format; +- HMAC token encoding, key provider, scope fields, or rotation procedure; +- reversible-token storage or cryptographic construction; +- production audit and mapping storage. diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md new file mode 100644 index 0000000..648afa0 --- /dev/null +++ b/docs/privacy-capability-matrix.md @@ -0,0 +1,46 @@ +# Privacy Capability Matrix + +DataFog Core preserves useful privacy capabilities without treating the Python +API as a contract. Each capability is classified by the treatment appropriate +for a PII detection and transformation engine. + +| Capability | Treatment | Core direction | +| --- | --- | --- | +| PII scanning | Preserve | Return validated findings with explicit byte and code-point ranges, optional confidence, and detector provenance. | +| Typed redaction | Preserve | Replace findings with typed, document-local placeholders after deterministic overlap resolution. | +| Character masking | Preserve | Support configurable masking with explicit direction and length semantics. | +| Entity-type selection | Preserve | Select canonical entity types through transformation configuration. | +| Exact allowlists | Preserve | Exempt exact values using documented, case-sensitive matching. | +| Regex allowlists | Preserve | Use full-match semantics, validated patterns, and bounded execution. | +| Locale selection | Preserve | Pass locale constraints to detectors without placing detector implementations in the transformation layer. | +| Precomputed findings | Preserve | Permit transformations over caller-supplied findings after strict validation. | +| Prompt/output guardrails | Out of scope | A governance layer may consume Core findings and results to make and enforce `allow`, `warn`, or `block` decisions. | +| Hash replacement | Redesign | Retain as a compatibility fingerprint with explicit equality and guessing leakage; do not call it pseudonymization. | +| Pseudonymization | Redesign | Implement a new keyed, scoped, deterministic, one-way value pseudonym. Do not copy numbered Python placeholders. | +| Reversible tokenization | New | Add opaque, authorized, reversible tokens through a key or vault boundary. | +| Restoration | New | Restore only known reversible tokens under explicit authorization and scope checks. | +| Transformation mappings | Redesign | Return ordered transformation records; make any mapping view explicit and sensitive. | +| Duplicate handling | Redesign | Collapse exact duplicates deterministically. | +| Overlap handling | Redesign | Resolve overlaps once in the transformation framework using documented precedence. | +| Malformed finding handling | Redesign | Reject the whole transformation instead of silently leaving possible PII unchanged. | +| Legacy function aliases | Compatibility only | Keep aliases out of the core; add wrappers only when a concrete migration need is accepted. | +| Legacy return shapes | Compatibility only | Reproduce only in a separately scoped compatibility binding, if needed. | +| Random fake-value replacement | Drop from core | Do not imply privacy guarantees for synthetic-looking replacements without a defined security and consistency contract. | +| โ€œAnonymizationโ€ claims | Drop | Describe concrete transformations and leakage; pseudonymized data is not anonymous. | +| spaCy, GLiNER, or engine selection | Out of scope | Detector implementations remain separate from the privacy transformation contract. | +| OCR and image extraction | Out of scope | Text extraction is a separate subsystem. | +| Dataset-level privacy | Out of scope | k-anonymity, generalization, synthetic datasets, and differential privacy require different data models and risk analysis. | + +## Compatibility evidence + +Python tests and outputs may be used in three ways: + +1. **Capability continuity:** prove that an important user workflow remains + possible. +2. **Exact compatibility:** reproduce an output only when a documented migration + requirement justifies it. +3. **Intentional divergence:** prove that the core behaves differently because + the new contract is safer or clearer. + +No abandoned or proposed Python versioned API is authoritative for the Rust +core. diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md new file mode 100644 index 0000000..02801ff --- /dev/null +++ b/docs/privacy-operations-roadmap.md @@ -0,0 +1,145 @@ +# PII Privacy Core Roadmap + +## Product framing + +DataFog Core detects sensitive information, applies privacy transformations, +and restores explicitly reversible tokens. Governance decisions and payload +enforcement belong to a separate layer. + +The target model is: + +```text +scan(text) -> findings +transform(...) -> transformed text and transformation records +restore(...) -> authorized restoration of reversible tokens +``` + +## Slice 0: Core contract + +**Status: complete** + +The accepted contract is recorded in +[ADR 001](adr/001-privacy-core-contract.md). The disposition of existing and +proposed capabilities is recorded in the +[privacy capability matrix](privacy-capability-matrix.md). + +The decisions cover: + +- the new-engine framing and compatibility stance; +- operation and strategy names; +- explicit offset coordinate systems; +- the finding shape; +- strict validation of supplied findings; +- deterministic duplicate and overlap handling; +- the transformation result shape; and +- the security meaning of hash, pseudonymize, and tokenize. + +## Slice 1: Finding and scan contract + +**Status: complete** + +- Introduce the accepted `Finding` representation. +- Preserve UTF-8 byte spans internally. +- Return byte and Unicode code-point ranges publicly. +- Add optional confidence and detector provenance. +- Update serialization and fixtures without changing detector coverage. + +**Proof:** existing detection fixtures remain valid; Unicode and binding +round-trip fixtures prove every public range selects the reported matched text. + +## Slice 2: Transformation framework and redaction + +- Validate caller-supplied findings strictly. +- Deduplicate and resolve overlaps once in shared code. +- Apply replacements without invalidating subsequent source ranges. +- Implement typed, document-local redaction placeholders. +- Return ordered transformation records and output ranges. + +**Proof:** normal, repeated, duplicate, overlapping, nested, empty, malformed, +and Unicode cases satisfy ADR 001. + +## Slice 3: Mask and remove + +- Define full and partial masking direction and length semantics. +- Support a validated masking character. +- Add literal removal as a small transformation if a concrete use case remains. + +**Proof:** transformation records identify the exact input and output spans for +full, partial, and zero-length replacements. + +## Slice 4: Transformation selection + +- Add entity-type selection. +- Add exact, case-sensitive allowlists. +- Add full-match regex allowlists with bounded behavior. +- Add locale selection and per-entity strategy overrides. +- Define transformation-configuration validation and precedence. + +**Proof:** the same configuration retains the same findings whether it scans +internally or receives precomputed findings. + +## Slice 5: Compatibility hash + +- Select and document the compatibility digest and encoding. +- Specify determinism, truncation, and collision behavior. +- Document equality leakage and low-entropy guessing risk. +- Keep hash distinct from secure pseudonymization. + +**Proof:** fixed vectors are stable and the API never describes the output as +anonymous or securely tokenized. + +## Slice 6: One-way pseudonymization + +- Define the key-provider boundary. +- Define required tenant, dataset, purpose, and scope-version context. +- Use a reviewed keyed construction with domain separation. +- Define token encoding and key rotation behavior. +- Suppress sensitive source mappings by default. + +**Proof:** identical values are stable within the same entity type and scope; +changing any scope component, entity type, or key version changes the output. + +## Slice 7: Reversible tokenization and restoration + +- Define opaque token format and authorization context. +- Introduce a vault or reviewed reversible cryptographic boundary. +- Implement token creation and restoration as one end-to-end slice. +- Reject unknown, expired, wrong-scope, and unauthorized tokens. + +**Proof:** authorized round trips succeed and every unauthorized variant fails +closed without revealing the original value. + +## Slice 8: Binding rollout + +After the Rust contract stabilizes: + +1. expose the API through the Python binding; +2. expose it through Node with explicit UTF-16 coordinate support; and +3. expose stateless operations through WASM. + +Reversible token storage is not promised in WASM without a separate key-custody +and storage design. + +## Acceptance bar + +Every completed slice must have: + +1. documented behavior; +2. normal, empty, duplicate, overlapping, malformed, and Unicode tests where + applicable; +3. security tests appropriate to the claimed property; +4. capability-continuity fixtures where Python demonstrates an important user + workflow; and +5. explicit documentation of intentional compatibility differences. + +## Explicitly out of scope + +- OCR and image extraction; +- spaCy, GLiNER, or other detector implementations; +- legacy Python APIs and return shapes unless separately approved; +- governance decisions and enforcement actions such as `allow`, `warn`, and + `block`; +- claims of complete anonymization; +- identity resolution across different identifiers; and +- dataset-level techniques such as k-anonymity, generalization, synthetic data, + and differential privacy. diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index 777eb11..e2e2322 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -36,6 +36,32 @@ const fixturesDirectory = process.argv[2]; assert.throws(() => scan(123), TypeError); +function legacyProjection(finding) { + return { + label: finding.entityType, + text: finding.matchedText, + start: finding.codepointRange.start, + end: finding.codepointRange.end, + }; +} + +function verifyContract(text, finding) { + const bytes = Buffer.from(text, "utf8"); + assert.equal( + bytes.subarray(finding.byteRange.start, finding.byteRange.end).toString("utf8"), + finding.matchedText, + ); + assert.equal( + Array.from(text) + .slice(finding.codepointRange.start, finding.codepointRange.end) + .join(""), + finding.matchedText, + ); + assert.equal(finding.confidence, undefined); + assert.ok(finding.detectorName.startsWith("datafog-core/")); + assert.ok(finding.detectorVersion); +} + for (const name of ["development.jsonl", "final.jsonl"]) { const records = readFileSync(path.join(fixturesDirectory, name), "utf8") .split("\\n") @@ -43,23 +69,35 @@ for (const name of ["development.jsonl", "final.jsonl"]) { .map((line) => JSON.parse(line)); for (const record of records) { - assert.deepEqual(scan(record.text), record.entities, \`\${name}: \${record.id}\`); + const findings = scan(record.text); + assert.deepEqual( + findings.map(legacyProjection), + record.entities, + \`\${name}: \${record.id}\`, + ); + findings.forEach((finding) => verifyContract(record.text, finding)); } } -console.log("Installed @datafog/node package matches both fixtures."); +const emojiFinding = scan("๐Ÿ‘‹ jane@example.com")[0]; +assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); +assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); + +console.log("Installed @datafog/node package matches fixtures and the Finding contract."); `.trimStart(), ); writeFileSync( path.join(temporaryDirectory, "type-smoke.ts"), ` -import { scan, type Entity, type Label } from "@datafog/node"; +import { scan, type EntityType, type Finding, type TextRange } from "@datafog/node"; -const entities: Entity[] = scan("Email jane@example.com"); -const label: Label = entities[0]?.label ?? "EMAIL"; +const findings: Finding[] = scan("Email jane@example.com"); +const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; +const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; -void label; +void entityType; +void range; `.trimStart(), ); diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index e40c08c..cc91e61 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -78,14 +78,16 @@ function writeConsumerFiles() { writeFileSync( path.join(temporaryDirectory, "type-smoke.ts"), ` -import { init, scan, type Entity, type Label } from "@datafog/wasm"; +import { init, scan, type EntityType, type Finding, type TextRange } from "@datafog/wasm"; const ready: Promise = init(); -const entities: Entity[] = scan("Email jane@example.com"); -const label: Label = entities[0]?.label ?? "EMAIL"; +const findings: Finding[] = scan("Email jane@example.com"); +const entityType: EntityType = findings[0]?.entityType ?? "CUSTOM_ENTITY"; +const range: TextRange = findings[0]?.byteRange ?? { start: 0, end: 0 }; void ready; -void label; +void entityType; +void range; `.trimStart(), ); @@ -177,6 +179,37 @@ try { await Promise.all([init(), init()]); expectThrows(() => scan(123), "TypeError"); + function legacyProjection(finding) { + return { + label: finding.entityType, + text: finding.matchedText, + start: finding.codepointRange.start, + end: finding.codepointRange.end, + }; + } + + function verifyContract(text, finding) { + const bytes = new TextEncoder().encode(text); + const matchedBytes = bytes.slice(finding.byteRange.start, finding.byteRange.end); + const matchedText = new TextDecoder().decode(matchedBytes); + if (matchedText !== finding.matchedText) { + throw new Error("byte range does not select matched text"); + } + + const matchedCodepoints = Array.from(text) + .slice(finding.codepointRange.start, finding.codepointRange.end) + .join(""); + if (matchedCodepoints !== finding.matchedText) { + throw new Error("code-point range does not select matched text"); + } + if (finding.confidence !== undefined) { + throw new Error("rule-based findings must omit confidence"); + } + if (!finding.detectorName.startsWith("datafog-core/") || !finding.detectorVersion) { + throw new Error("detector provenance is missing"); + } + } + for (const fixture of ["development.jsonl", "final.jsonl"]) { const source = await fetch(`/${fixture}`).then((response) => response.text()); const records = source @@ -185,14 +218,27 @@ try { .map((line) => JSON.parse(line)); for (const record of records) { - if (JSON.stringify(scan(record.text)) !== JSON.stringify(record.entities)) { + const findings = scan(record.text); + if ( + JSON.stringify(findings.map(legacyProjection)) !== + JSON.stringify(record.entities) + ) { throw new Error(`${fixture}: ${record.id}`); } + findings.forEach((finding) => verifyContract(record.text, finding)); } } + + const emojiFinding = scan("๐Ÿ‘‹ jane@example.com")[0]; + if ( + JSON.stringify(emojiFinding.byteRange) !== JSON.stringify({ start: 5, end: 21 }) || + JSON.stringify(emojiFinding.codepointRange) !== JSON.stringify({ start: 2, end: 18 }) + ) { + throw new Error("Unicode ranges do not use the documented coordinate systems"); + } }); - console.log("Installed @datafog/wasm package matches both fixtures."); + console.log("Installed @datafog/wasm package matches fixtures and the Finding contract."); } finally { await browser?.close(); await new Promise((resolve) => server?.close(resolve) ?? resolve());