From 37584dd8d167e61dba6dd5a7a31d060463289692 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:28:52 -0700 Subject: [PATCH 1/3] docs: exclude unkeyed hash from privacy core --- docs/adr/001-privacy-core-contract.md | 12 ++++--- docs/privacy-capability-matrix.md | 2 +- docs/privacy-operations-roadmap.md | 50 +++++++++++++++++++-------- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index 17d62cf..663f167 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -31,7 +31,6 @@ Canonical transformation strategies are: remove redact mask -hash pseudonymize tokenize ``` @@ -356,8 +355,14 @@ original values are excluded from default debug and log output. configuration, does not consume surrounding whitespace, and does not normalize the remaining text. Its transformation record uses an empty output range at the deletion position. -- `hash` is a compatibility fingerprint with explicitly documented leakage and - must not be presented as secure pseudonymization. +- Unkeyed `hash` is not a canonical Core transformation. Predictable PII can + be tested through brute-force or dictionary attacks, while deterministic + output exposes equality and cross-dataset linkage. Salting does not provide + a useful middle ground: a public or shared salt remains guessable, a random + per-value salt removes deterministic matching, and a secret salt is a keyed + pseudonymization design. A separately scoped compatibility adapter may offer + a plainly named fingerprint only when a concrete migration requirement + justifies it. - `pseudonymize` is a new keyed, scoped, deterministic, one-way operation. It is not based on the Python numbered-placeholder behavior. - `tokenize` creates opaque reversible or vault-backed tokens. @@ -391,7 +396,6 @@ and documented separately. This ADR does not choose: -- compatibility hash format; - HMAC token encoding, key provider, scope fields, or rotation procedure; - reversible-token storage or cryptographic construction; - production audit and mapping storage. diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md index d07b5c7..6d194b0 100644 --- a/docs/privacy-capability-matrix.md +++ b/docs/privacy-capability-matrix.md @@ -15,7 +15,7 @@ for a PII detection and transformation engine. | Locale selection | Preserve | Pass locale constraints to detectors without placing detector implementations in the transformation layer. | | Precomputed findings | Preserve | Permit transformations over caller-supplied findings after strict validation. | | Prompt/output guardrails | Out of scope | A governance layer may consume Core findings and results to make and enforce `allow`, `warn`, or `block` decisions. | -| Hash replacement | Redesign | Retain as a compatibility fingerprint with explicit equality and guessing leakage; do not call it pseudonymization. | +| Hash replacement | Compatibility only | Exclude unkeyed hashing from canonical Core transformations; consider a plainly named fingerprint in a separate compatibility layer only for an accepted migration requirement. | | Pseudonymization | Redesign | Implement a new keyed, scoped, deterministic, one-way value pseudonym. Do not copy numbered Python placeholders. | | Reversible tokenization | New | Add opaque, authorized, reversible tokens through a key or vault boundary. | | Restoration | New | Restore only known reversible tokens under explicit authorization and scope checks. | diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index a462c24..19555a6 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -33,7 +33,7 @@ The decisions cover: - strict validation of supplied findings; - deterministic duplicate and overlap handling; - the transformation result shape; and -- the security meaning of hash, pseudonymize, and tokenize. +- the security meaning and disposition of hash, pseudonymize, and tokenize. ## Slice 1: Finding and scan contract @@ -83,6 +83,8 @@ replacements. Invalid strategy fields and masking characters are rejected. ## Slice 4: Transformation selection +**Status: complete** + - Replace the temporary single-strategy request with one canonical transformation-configuration envelope. Do not retain the old shape as a shorthand or add a second transformation operation. @@ -121,15 +123,22 @@ override fallback, dormant rules, malformed configuration, and Unicode cases produce the same result whether findings come directly from `scan` or are supplied to `transform`. -## Slice 5: Compatibility hash +## Slice 5: Exclude unkeyed hash -- Select and document the compatibility digest and encoding. -- Specify determinism, truncation, and collision behavior. -- Document equality leakage and low-entropy guessing risk. -- Keep hash distinct from secure pseudonymization. +**Status: complete** -**Proof:** fixed vectors are stable and the API never describes the output as -anonymous or securely tokenized. +- Do not expose unkeyed hashing as a canonical privacy transformation. +- Treat brute-force guessing and deterministic linkage as disqualifying for + predictable PII, regardless of digest length or encoding. +- Do not add salt configuration as a compromise: public salts remain guessable, + per-value random salts remove stable equality, and secret salts belong to the + keyed pseudonymization design. +- Permit a non-Core compatibility fingerprint only when a concrete migration + requirement is separately accepted and documented. + +**Proof:** the canonical strategy set contains no unkeyed hash operation and +documentation directs deterministic one-way privacy requirements to scoped, +keyed pseudonymization. ## Slice 6: One-way pseudonymization @@ -152,16 +161,27 @@ changing any scope component, entity type, or key version changes the output. **Proof:** authorized round trips succeed and every unauthorized variant fails closed without revealing the original value. -## Slice 8: Binding rollout +## Slice 8: Binding completion and release hardening + +Python, Node, and WASM already expose Slices 1 through 4. New stateless +operations should continue to ship through those bindings in the same vertical +slice as their Rust implementation rather than waiting for a separate binding +rollout. -After the Rust contract stabilizes: +Remaining binding work is: -1. expose the API through the Python binding; -2. expose it through Node with explicit UTF-16 coordinate support; and -3. expose stateless operations through WASM. +1. add explicitly named UTF-16 code-unit ranges for JavaScript consumers as + required by ADR 001, without changing the existing byte or code-point + fields; +2. carry one-way pseudonymization through Python, Node, and WASM when that + slice is implemented; +3. expose reversible tokenization and restoration only through runtimes with a + separately accepted key-custody, authorization, and storage boundary; and +4. retain installed-package and cross-binding conformance tests as release + gates. -Reversible token storage is not promised in WASM without a separate key-custody -and storage design. +Reversible token storage is not promised in WASM without that separate +key-custody and storage design. ## Acceptance bar From d923eb072451cbe548c0eaed8e59f11d4e23979c Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:03:37 -0700 Subject: [PATCH 2/3] docs: canonize one-way pseudonymization contract --- docs/adr/001-privacy-core-contract.md | 107 ++++++++++++++++++++++---- docs/privacy-capability-matrix.md | 4 +- docs/privacy-operations-roadmap.md | 43 ++++++++--- 3 files changed, 127 insertions(+), 27 deletions(-) diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index 663f167..e27aa9d 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -47,6 +47,11 @@ with typed enum variants; object-oriented bindings serialize it as: character: "*", reveal: { direction: "first" | "last", count: non_negative_integer } } +{ + strategy: "pseudonymize", + key_ref: "provider-specific-key-reference", + key_version?: "provider-specific-version-or-alias" +} ``` Fields that do not belong to the selected strategy are rejected rather than @@ -193,6 +198,13 @@ Node, and WASM. The stable top-level error codes are: invalid_configuration invalid_finding internal_error +key_provider_required +key_not_found +key_access_denied +key_provider_unavailable +invalid_key_material +key_provider_error +unsupported_strategy ``` Caller-correctable errors also include a stable machine-readable `reason` and @@ -231,6 +243,16 @@ native exception hierarchy, but the canonical fields retain the same meaning. Additional reason values may be introduced as validation expands without creating new top-level categories for every validation case. +Key-provider errors never contain key bytes, source text, credentials, or the +key reference. Their path identifies the pseudonymization selector that could +not be resolved. `key_provider_required` means a synchronous providerless call +selected pseudonymization. `key_provider_unavailable` indicates that retrying +the entire transformation may succeed; no other provider error is +automatically retryable. `unsupported_strategy` identifies a strategy that a +binding intentionally cannot execute, including pseudonymization in browser +WASM. Core does not add retries or backoff around a provider. Provider +implementations own network timeouts and any SDK-level retry policy. + `allow`, `warn`, and `block` are governance decisions, not Core operations or transformation strategies. A separate governance layer may consume findings and transformation results to make those decisions. The calling application, @@ -320,22 +342,31 @@ TransformResult { } Transformation { - finding + entity_type + source_byte_range + source_codepoint_range + confidence? + detector_name + detector_version? strategy replacement output_byte_range output_codepoint_range + key_ref? // pseudonymize only + resolved_key_version? // pseudonymize only } ``` Transformation records are ordered by source document position and include only transformations that were actually applied. Output ranges are zero-based, end-exclusive, refer to the transformed text, and select exactly `replacement`. -The finding already supplies the original value and source ranges, so the -canonical payload does not contain a second original-to-replacement mapping. +Source metadata deliberately excludes `matched_text`; Core does not echo the +original PII or offer an include-originals switch. Callers that explicitly need +the source value already possess the input and can use the source ranges. -A mapping view may be offered as an explicit convenience API. Sensitive -original values are excluded from default debug and log output. +Pseudonymization records include the configured key reference and the concrete +version returned by the provider. They never include key material or a +plaintext-to-token mapping. Non-pseudonymization records omit both key fields. ### Security meaning of strategies @@ -363,8 +394,13 @@ original values are excluded from default debug and log output. pseudonymization design. A separately scoped compatibility adapter may offer a plainly named fingerprint only when a concrete migration requirement justifies it. -- `pseudonymize` is a new keyed, scoped, deterministic, one-way operation. It - is not based on the Python numbered-placeholder behavior. +- `pseudonymize` is a new keyed, deterministic, one-way operation. It is not + based on the Python numbered-placeholder behavior. It computes HMAC-SHA-256 + over the exact UTF-8 bytes of `matched_text` and encodes the complete 32-byte + digest as standard padded Base64. The result is always 44 characters. Core + performs no trimming, case folding, Unicode normalization, semantic + canonicalization, domain separation, digest truncation, or algorithm + negotiation. - `tokenize` creates opaque reversible or vault-backed tokens. - `restore` accepts only explicitly reversible tokens and requires authorization. @@ -372,6 +408,51 @@ original values are excluded from default debug and log output. Pseudonymization is value pseudonymization, not identity resolution. The core does not infer that different identifiers belong to the same person. +### Pseudonymization key contract + +The key defines the linkage scope. The same exact value and key material +produce the same pseudonym, including across different entity types. Separate +tenants, datasets, or purposes use separate keys or key versions; those +concepts are not additional Core configuration fields. + +Serializable pseudonymization configuration contains a required `key_ref` and +an optional `key_version`. A runtime `KeyProvider` receives those identifiers +and returns exactly 32 cryptographically random key bytes plus a non-empty +concrete resolved version. Core rejects every other key length and never pads, +hashes, or derives arbitrary input into a key. Base64 decoding, secret-store +formats, password-based derivation, and provider authentication remain outside +Core. + +Key resolution is separated from the synchronous transformation kernel. An +asynchronous `PrivacyManager` owns the provider, resolves every distinct key +selector required by the selected findings exactly once, and freezes all +resolved versions for the request. It resolves no keys for findings removed by +entity selection, allowlists, duplicate handling, or overlap resolution. All +keys must resolve and validate before text mutation begins; one failure returns +no text or transformation records. An explicitly requested version never +falls back to another version. Omitting a version permits provider-defined +latest-version behavior, so a later whole-operation retry may observe a +rotation and produce different pseudonyms. + +Default and entity-specific strategies may refer to different keys. Distinct +selectors are deduplicated within one request, and each applied record reports +the key reference and concrete resolved version it used. + +Core holds resolved material in a non-clonable, non-serializable, +redacted-debug container for one call, never caches it across calls, and +best-effort zeroizes it on every exit path. A provider may implement explicit +caching but owns its TTL and rotation behavior. Core never logs key material or +places it in an error. + +Rust, Python, and Node expose the provider-backed manager. The existing +synchronous operations remain available for `remove`, `redact`, and `mask`; +attempting to apply `pseudonymize` without a manager fails with a structured +provider-required error. Browser WASM pseudonymization is deferred because the +chosen provider model would place raw key bytes in browser-accessible linear +memory. Core ships no cloud-vendor SDK adapter in this slice; AWS, Google, and +other integrations belong in separate packages behind the same provider +contract. + ## Capability-continuity stance Python behavior is classified as preserved, redesigned, compatibility-only, or @@ -385,10 +466,11 @@ and documented separately. portable. - Strict finding validation may intentionally differ from permissive legacy behavior. -- Consumers receive auditable transformation records without a redundant - mapping dictionary. -- Secure pseudonymization and tokenization require explicit key and scope - contracts in later ADRs. +- Consumers receive auditable transformation records without echoing original + PII or returning a mapping dictionary. +- Secure pseudonymization requires an explicit key-provider contract; + reversible tokenization requires a separately accepted key, authorization, + and storage boundary. - Pseudonymized data remains sensitive and must not be described as anonymous. - Governance decisions and payload enforcement remain outside DataFog Core. @@ -396,7 +478,6 @@ and documented separately. This ADR does not choose: -- HMAC token encoding, key provider, scope fields, or rotation procedure; - reversible-token storage or cryptographic construction; -- production audit and mapping storage. +- production audit storage; or - custom literal replacement or whitespace-normalizing removal. diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md index 6d194b0..9cc39fc 100644 --- a/docs/privacy-capability-matrix.md +++ b/docs/privacy-capability-matrix.md @@ -16,10 +16,10 @@ for a PII detection and transformation engine. | Precomputed findings | Preserve | Permit transformations over caller-supplied findings after strict validation. | | Prompt/output guardrails | Out of scope | A governance layer may consume Core findings and results to make and enforce `allow`, `warn`, or `block` decisions. | | Hash replacement | Compatibility only | Exclude unkeyed hashing from canonical Core transformations; consider a plainly named fingerprint in a separate compatibility layer only for an accepted migration requirement. | -| Pseudonymization | Redesign | Implement a new keyed, scoped, deterministic, one-way value pseudonym. Do not copy numbered Python placeholders. | +| Pseudonymization | Redesign | Use HMAC-SHA-256 over exact UTF-8 input with a provider-resolved 256-bit key and full padded-Base64 output. The key defines linkage scope; do not copy numbered Python placeholders. | | Reversible tokenization | New | Add opaque, authorized, reversible tokens through a key or vault boundary. | | Restoration | New | Restore only known reversible tokens under explicit authorization and scope checks. | -| Transformation mappings | Redesign | Return ordered transformation records; make any mapping view explicit and sensitive. | +| Transformation mappings | Redesign | Return ordered transformation records without `matched_text` or a plaintext-to-token mapping. Preserve source ranges and non-sensitive audit metadata. | | Duplicate handling | Redesign | Collapse exact duplicates deterministically. | | Overlap handling | Redesign | Resolve overlaps once in the transformation framework using documented precedence. | | Malformed finding handling | Redesign | Reject the whole transformation instead of silently leaving possible PII unchanged. | diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 19555a6..5164040 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -142,14 +142,33 @@ keyed pseudonymization. ## Slice 6: One-way pseudonymization -- Define the key-provider boundary. -- Define required tenant, dataset, purpose, and scope-version context. -- Use a reviewed keyed construction with domain separation. -- Define token encoding and key rotation behavior. -- Suppress sensitive source mappings by default. - -**Proof:** identical values are stable within the same entity type and scope; -changing any scope component, entity type, or key version changes the output. +**Status: contract accepted; implementation pending** + +- Add `pseudonymize` with required `key_ref` and optional `key_version`. +- Use fixed HMAC-SHA-256 over the exact UTF-8 matched value and encode the full + digest as 44-character standard padded Base64. +- Let the key define linkage scope. Do not add tenant, dataset, purpose, + entity-type, domain-separation, normalization, or algorithm-selection fields. +- Resolve provider keys asynchronously before entering the synchronous + transformation kernel. Require exactly 32 random bytes and a concrete + resolved version. +- Resolve each distinct selector used by selected findings once, freeze all + versions for the request, and fail atomically if any resolution fails. +- Keep key material out of serialized configuration, logs, debug output, + errors, and results; retain it for one call only and best-effort zeroize it. +- Permit provider-owned caching and retries without adding either to Core. +- Remove `matched_text` from every transformation record. Preserve source + ranges and detector metadata, and report `key_ref` plus concrete key version + only for pseudonymization records. +- Ship the provider-backed manager through Rust, Python, and Node. Defer + browser WASM pseudonymization and cloud-specific provider adapters. + +**Proof:** exact input and the same key produce the same full HMAC token across +Rust, Python, and Node; changing exact input or resolved key material changes +the token; different entity types do not alter it; multiple selectors resolve +once and apply atomically; provider failures, invalid key material, and +providerless or browser-WASM calls fail closed; no transformation record echoes +the original PII. ## Slice 7: Reversible tokenization and restoration @@ -173,15 +192,15 @@ Remaining binding work is: 1. add explicitly named UTF-16 code-unit ranges for JavaScript consumers as required by ADR 001, without changing the existing byte or code-point fields; -2. carry one-way pseudonymization through Python, Node, and WASM when that - slice is implemented; +2. retain Rust, Python, and Node pseudonymization conformance coverage while + keeping browser WASM key handling explicitly unsupported; 3. expose reversible tokenization and restoration only through runtimes with a separately accepted key-custody, authorization, and storage boundary; and 4. retain installed-package and cross-binding conformance tests as release gates. -Reversible token storage is not promised in WASM without that separate -key-custody and storage design. +Pseudonymization and reversible token storage are not promised in browser WASM +without a separately accepted host-managed key-custody boundary. ## Acceptance bar From 481b713c34e12e0c259cabef869c73ae3a3ae759 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:20:01 -0700 Subject: [PATCH 3/3] feat: implement provider-backed pseudonymization --- Cargo.lock | 128 ++++ README.md | 44 +- bindings/node/dts-header.d.ts | 45 +- bindings/node/index.d.ts | 78 ++- bindings/node/index.js | 131 ++++ bindings/node/src/lib.rs | 164 ++++- bindings/python/Cargo.toml | 1 + bindings/python/src/lib.rs | 198 +++++- bindings/python/tests/test_installed.py | 60 ++ bindings/wasm/index.d.ts | 16 +- bindings/wasm/src/lib.rs | 42 +- crates/core/Cargo.toml | 7 + crates/core/src/lib.rs | 866 +++++++++++++++++++++++- docs/privacy-operations-roadmap.md | 2 +- scripts/test-node-package.mjs | 65 +- scripts/test-wasm-package.mjs | 23 + 16 files changed, 1819 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d659b1..60376ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,12 +11,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -38,6 +53,25 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctor" version = "1.0.13" @@ -48,9 +82,14 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" name = "datafog-core" version = "0.1.0" dependencies = [ + "base64", + "futures", + "hmac", "regex", "serde", "serde_json", + "sha2", + "zeroize", ] [[package]] @@ -59,6 +98,7 @@ version = "0.1.0" dependencies = [ "datafog-core", "pyo3", + "pyo3-async-runtimes", "serde_json", ] @@ -84,6 +124,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "futures" version = "0.3.34" @@ -172,12 +223,31 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "itoa" version = "1.0.18" @@ -324,6 +394,20 @@ dependencies = [ "pyo3-macros", ] +[[package]] +name = "pyo3-async-runtimes" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" +dependencies = [ + "futures-channel", + "futures-util", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" version = "0.29.2" @@ -477,12 +561,29 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -511,6 +612,21 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "pin-project-lite", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -523,6 +639,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -574,6 +696,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zmij" version = "1.0.23" diff --git a/README.md b/README.md index cbaaea5..e8870c3 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,18 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based detectors currently report no confidence score. -The initial transformation strategies are `redact`, `mask`, and `remove`. +The transformation strategies are `redact`, `mask`, `remove`, and +`pseudonymize` in Rust, Python, and Node.js. Redaction uses an unnumbered `[ENTITY_TYPE]` placeholder, masking supports full or leading/trailing reveal modes, and removal deletes only the exact finding -span. `transform` requires explicit findings; `scan_and_transform` (or +span. Pseudonymization uses provider-resolved 256-bit keys and deterministic +HMAC-SHA-256 tokens; it is deliberately unsupported in browser WASM. +`transform` requires explicit findings; `scan_and_transform` (or `scanAndTransform` in JavaScript) is the explicit scan-then-transform convenience. Results include the transformed text and an ordered record for -every applied replacement, including its output byte and code-point ranges. +every applied replacement, including its source metadata and output byte and +code-point ranges. Transformation records never include the original matched +text. Transformation calls require an envelope with a default strategy. It can also select entity types, override the strategy per entity, and exempt exact or @@ -86,7 +91,9 @@ python -m pip install datafog-core ``` ```python -from datafog_core import scan, scan_and_transform +import asyncio + +from datafog_core import PrivacyManager, scan, scan_and_transform findings = scan("Email jane@example.com") print(findings[0].entity_type) # EMAIL @@ -111,6 +118,22 @@ masked = scan_and_transform( }, ) assert masked.text == "Email ************.com" + +class KeyProvider: + async def resolve_key(self, key_ref, key_version): + return {"key": load_32_byte_key(key_ref, key_version), "resolved_version": "7"} + +async def pseudonymize(): + return await PrivacyManager(KeyProvider()).scan_and_transform( + "Email jane@example.com", + { + "transform": { + "default": {"strategy": "pseudonymize", "key_ref": "customers/email"} + } + }, + ) + +pseudonymized = asyncio.run(pseudonymize()) ``` ### Node.js @@ -118,7 +141,7 @@ assert masked.text == "Email ************.com" `@datafog/node` will install as a native package once its npm release is published. ```js -import { scan, scanAndTransform } from "@datafog/node"; +import { PrivacyManager, scan, scanAndTransform } from "@datafog/node"; console.log(scan("Email jane@example.com")); console.log( @@ -126,6 +149,17 @@ console.log( transform: { default: { strategy: "redact" } }, }).text, ); + +const manager = new PrivacyManager({ + async resolveKey({ keyRef, keyVersion }) { + return { key: await load32ByteKey(keyRef, keyVersion), resolvedVersion: "7" }; + }, +}); +const pseudonymized = await manager.scanAndTransform("Email jane@example.com", { + transform: { + default: { strategy: "pseudonymize", key_ref: "customers/email" }, + }, +}); ``` The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64. diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index f413c4c..b73e5bf 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -1,7 +1,11 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; -export type TransformationStrategy = "redact" | "mask" | "remove"; +export type TransformationStrategy = + | "redact" + | "mask" + | "remove" + | "pseudonymize"; export interface MaskRevealConfig { readonly direction: "first" | "last"; @@ -11,6 +15,11 @@ export interface MaskRevealConfig { export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } + | { + readonly strategy: "pseudonymize"; + readonly key_ref: string; + readonly key_version?: string; + } | { readonly strategy: "mask"; readonly character?: string; @@ -46,6 +55,13 @@ export interface ScanAndTransformConfig { export type DataFogErrorCode = | "invalid_configuration" | "invalid_finding" + | "key_provider_required" + | "key_not_found" + | "key_access_denied" + | "key_provider_unavailable" + | "invalid_key_material" + | "key_provider_error" + | "unsupported_strategy" | "internal_error"; export declare class DataFogError extends Error { @@ -54,3 +70,30 @@ export declare class DataFogError extends Error { readonly path?: string; readonly findingIndex?: number; } + +export interface KeyProviderRequest { + readonly keyRef: string; + readonly keyVersion?: string; +} + +export interface KeyProviderResponse { + readonly key: Uint8Array; + readonly resolvedVersion: string; +} + +export interface KeyProvider { + resolveKey(request: KeyProviderRequest): Promise; +} + +export declare class PrivacyManager { + constructor(provider: KeyProvider); + transform( + text: string, + findings: Finding[], + config: TransformationConfig, + ): Promise; + scanAndTransform( + text: string, + config: ScanAndTransformConfig, + ): Promise; +} diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index a47594a..3fd38a7 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -1,7 +1,11 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; -export type TransformationStrategy = "redact" | "mask" | "remove"; +export type TransformationStrategy = + | "redact" + | "mask" + | "remove" + | "pseudonymize"; export interface MaskRevealConfig { readonly direction: "first" | "last"; @@ -11,6 +15,11 @@ export interface MaskRevealConfig { export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } + | { + readonly strategy: "pseudonymize"; + readonly key_ref: string; + readonly key_version?: string; + } | { readonly strategy: "mask"; readonly character?: string; @@ -46,6 +55,13 @@ export interface ScanAndTransformConfig { export type DataFogErrorCode = | "invalid_configuration" | "invalid_finding" + | "key_provider_required" + | "key_not_found" + | "key_access_denied" + | "key_provider_unavailable" + | "invalid_key_material" + | "key_provider_error" + | "unsupported_strategy" | "internal_error"; export declare class DataFogError extends Error { @@ -54,6 +70,33 @@ export declare class DataFogError extends Error { readonly path?: string; readonly findingIndex?: number; } + +export interface KeyProviderRequest { + readonly keyRef: string; + readonly keyVersion?: string; +} + +export interface KeyProviderResponse { + readonly key: Uint8Array; + readonly resolvedVersion: string; +} + +export interface KeyProvider { + resolveKey(request: KeyProviderRequest): Promise; +} + +export declare class PrivacyManager { + constructor(provider: KeyProvider); + transform( + text: string, + findings: Finding[], + config: TransformationConfig, + ): Promise; + scanAndTransform( + text: string, + config: ScanAndTransformConfig, + ): Promise; +} export interface Finding { readonly entityType: EntityType readonly matchedText: string @@ -64,6 +107,28 @@ export interface Finding { readonly detectorVersion?: string } +export interface KeySelector { + readonly index: number + readonly keyRef: string + readonly keyVersion?: string + readonly path: string +} + +export interface PreparedScanAndTransform { + readonly findings: Array + readonly selectors: Array +} + +export declare function prepareScanAndTransform(text: string, config: ScanAndTransformConfig): PreparedScanAndTransform + +export declare function requiredKeySelectors(text: string, findings: Array, config: TransformationConfig): Array + +export interface ResolvedKeyInput { + selectorIndex: number + key: Uint8Array + resolvedVersion: string +} + /** Scan text for supported PII findings. */ export declare function scan(text: string, config?: ScanConfig | undefined): Array @@ -79,14 +144,23 @@ export interface TextRange { export declare function transform(text: string, findings: Array, config: TransformationConfig): TransformResult export interface Transformation { - readonly finding: Finding + readonly entityType: string + readonly sourceByteRange: TextRange + readonly sourceCodepointRange: TextRange + readonly confidence?: number + readonly detectorName: string + readonly detectorVersion?: string readonly strategy: TransformationStrategy readonly replacement: string readonly outputByteRange: TextRange readonly outputCodepointRange: TextRange + readonly keyRef?: string + readonly resolvedKeyVersion?: string } export interface TransformResult { readonly text: string readonly transformations: Array } + +export declare function transformWithResolvedKeys(text: string, findings: Array, config: TransformationConfig, resolvedKeys: Array): TransformResult diff --git a/bindings/node/index.js b/bindings/node/index.js index b82adae..0f52c25 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,7 +1,11 @@ +import { Buffer } from "node:buffer"; import { + prepareScanAndTransform as nativePrepareScanAndTransform, + requiredKeySelectors as nativeRequiredKeySelectors, scan as nativeScan, scanAndTransform as nativeScanAndTransform, transform as nativeTransform, + transformWithResolvedKeys as nativeTransformWithResolvedKeys, } from "./native.js"; export class DataFogError extends Error { @@ -36,6 +40,133 @@ function normalizeError(error, fallbackCode) { }); } +const providerErrorMessages = { + key_not_found: "key provider could not find the requested key", + key_access_denied: "key provider denied access to the requested key", + key_provider_unavailable: "key provider is temporarily unavailable", + key_provider_error: "key provider could not resolve the requested key", +}; + +function normalizeProviderError(error, path) { + const candidateCode = error?.code; + const code = + typeof candidateCode === "string" && candidateCode in providerErrorMessages + ? candidateCode + : "key_provider_error"; + return new DataFogError({ + code, + message: providerErrorMessages[code], + path, + }); +} + +function withPathPrefix(error, prefix) { + const normalized = normalizeError(error, "internal_error"); + return new DataFogError({ + code: normalized.code, + reason: normalized.reason, + message: normalized.message, + path: normalized.path ? `${prefix}${normalized.path}` : normalized.path, + findingIndex: normalized.findingIndex, + }); +} + +function assertProvider(provider) { + if (!provider || typeof provider.resolveKey !== "function") { + throw new TypeError("PrivacyManager provider must define resolveKey(request)"); + } +} + +function resolvedKeyInput(selector, response) { + if ( + !response || + !(response.key instanceof Uint8Array) || + typeof response.resolvedVersion !== "string" + ) { + return { + selectorIndex: selector.index, + key: Buffer.alloc(0), + resolvedVersion: "", + }; + } + return { + selectorIndex: selector.index, + key: Buffer.from(response.key), + resolvedVersion: response.resolvedVersion, + }; +} + +export class PrivacyManager { + #provider; + + constructor(provider) { + assertProvider(provider); + this.#provider = provider; + } + + async #resolve(selectors, pathPrefix = "") { + const resolved = []; + for (const selector of selectors) { + let response; + try { + response = await this.#provider.resolveKey({ + keyRef: selector.keyRef, + keyVersion: selector.keyVersion, + }); + } catch (error) { + throw normalizeProviderError(error, `${pathPrefix}${selector.path}`); + } + resolved.push(resolvedKeyInput(selector, response)); + } + return resolved; + } + + async transform(text, findings, config) { + if (typeof text !== "string") { + throw new TypeError("transform text must be a string"); + } + if (!Array.isArray(findings)) { + throw new TypeError("transform findings must be an array"); + } + let resolved = []; + try { + const selectors = nativeRequiredKeySelectors(text, findings, config); + resolved = await this.#resolve(selectors); + return nativeTransformWithResolvedKeys(text, findings, config, resolved); + } catch (error) { + if (error instanceof DataFogError) throw error; + throw normalizeError(error, "invalid_configuration"); + } finally { + resolved.forEach(({ key }) => key.fill(0)); + } + } + + async scanAndTransform(text, config) { + if (typeof text !== "string") { + throw new TypeError("scanAndTransform text must be a string"); + } + let prepared; + try { + prepared = nativePrepareScanAndTransform(text, config); + } catch (error) { + throw normalizeError(error, "invalid_configuration"); + } + const resolved = await this.#resolve(prepared.selectors, "/transform"); + try { + return nativeTransformWithResolvedKeys( + text, + prepared.findings, + config.transform, + resolved, + ); + } catch (error) { + throw withPathPrefix(error, "/transform"); + } finally { + resolved.forEach(({ key }) => key.fill(0)); + } + } +} + export function scan(text, config) { if (typeof text !== "string") { throw new TypeError("scan text must be a string"); diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index ac07a10..90e2ad9 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -1,5 +1,6 @@ //! Node binding for datafog-core. +use napi::bindgen_prelude::Buffer; use napi::{Env, Error, Status, Unknown}; use napi_derive::napi; @@ -39,7 +40,22 @@ pub struct Finding { #[napi(object, object_from_js = false)] pub struct Transformation { #[napi(readonly)] - pub finding: Finding, + pub entity_type: String, + + #[napi(readonly)] + pub source_byte_range: TextRange, + + #[napi(readonly)] + pub source_codepoint_range: TextRange, + + #[napi(readonly)] + pub confidence: Option, + + #[napi(readonly)] + pub detector_name: String, + + #[napi(readonly)] + pub detector_version: Option, #[napi(readonly, ts_type = "TransformationStrategy")] pub strategy: String, @@ -52,6 +68,12 @@ pub struct Transformation { #[napi(readonly)] pub output_codepoint_range: TextRange, + + #[napi(readonly)] + pub key_ref: Option, + + #[napi(readonly)] + pub resolved_key_version: Option, } #[napi(object, object_from_js = false)] @@ -63,6 +85,38 @@ pub struct TransformResult { pub transformations: Vec, } +#[napi(object, object_from_js = false)] +pub struct KeySelector { + #[napi(readonly)] + pub index: u32, + + #[napi(readonly)] + pub key_ref: String, + + #[napi(readonly)] + pub key_version: Option, + + #[napi(readonly)] + pub path: String, +} + +#[napi(object)] +pub struct ResolvedKeyInput { + pub selector_index: u32, + #[napi(ts_type = "Uint8Array")] + pub key: Buffer, + pub resolved_version: String, +} + +#[napi(object, object_from_js = false)] +pub struct PreparedScanAndTransform { + #[napi(readonly)] + pub findings: Vec, + + #[napi(readonly)] + pub selectors: Vec, +} + fn js_offset(offset: usize) -> napi::Result { u32::try_from(offset).map_err(|_| { Error::new( @@ -118,7 +172,9 @@ fn js_privacy_error(error: datafog_core::PrivacyError) -> Error { "findingIndex": error.finding_index(), }); let status = match error.code() { - datafog_core::PrivacyErrorCode::InternalError => Status::GenericFailure, + datafog_core::PrivacyErrorCode::InternalError + | datafog_core::PrivacyErrorCode::KeyProviderUnavailable + | datafog_core::PrivacyErrorCode::KeyProviderError => Status::GenericFailure, _ => Status::InvalidArg, }; Error::new(status, payload.to_string()) @@ -132,21 +188,67 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result "redact".to_owned(), datafog_core::TransformationStrategy::Remove => "remove".to_owned(), datafog_core::TransformationStrategy::Mask(_) => "mask".to_owned(), + datafog_core::TransformationStrategy::Pseudonymize(_) => { + "pseudonymize".to_owned() + } }, replacement: transformation.replacement, output_byte_range: js_range(transformation.output_byte_range)?, output_codepoint_range: js_range(transformation.output_codepoint_range)?, + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, }) }) .collect::>>()?, }) } +fn js_key_selectors(selectors: &[datafog_core::KeySelector]) -> napi::Result> { + selectors + .iter() + .enumerate() + .map(|(index, selector)| { + Ok(KeySelector { + index: js_offset(index)?, + key_ref: selector.key_ref().to_owned(), + key_version: selector.key_version().map(str::to_owned), + path: selector.path().to_owned(), + }) + }) + .collect() +} + +fn core_key_bindings( + selectors: Vec, + resolved_keys: Vec, +) -> napi::Result> { + resolved_keys + .into_iter() + .map(|resolved| { + let selector = selectors + .get(resolved.selector_index as usize) + .cloned() + .ok_or_else(|| { + Error::new(Status::InvalidArg, "resolved key selector index is invalid") + })?; + Ok(datafog_core::ResolvedKeyBinding::new( + selector, + datafog_core::ResolvedKey::new(resolved.key.to_vec(), resolved.resolved_version), + )) + }) + .collect() +} + /// Scan text for supported PII findings. #[napi(strict, catch_unwind)] pub fn scan( @@ -196,3 +298,59 @@ pub fn scan_and_transform( .map_err(js_privacy_error) .and_then(js_transform_result) } + +#[napi(strict, catch_unwind)] +pub fn required_key_selectors( + env: Env, + text: String, + findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, +) -> napi::Result> { + let config: serde_json::Value = env.from_js_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?; + let findings = findings.into_iter().map(core_finding).collect::>(); + let selectors = datafog_core::required_key_selectors(&text, &findings, &config) + .map_err(js_privacy_error)?; + js_key_selectors(&selectors) +} + +#[napi(strict, catch_unwind)] +pub fn transform_with_resolved_keys( + env: Env, + text: String, + findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, + resolved_keys: Vec, +) -> napi::Result { + let config: serde_json::Value = env.from_js_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?; + let findings = findings.into_iter().map(core_finding).collect::>(); + let selectors = datafog_core::required_key_selectors(&text, &findings, &config) + .map_err(js_privacy_error)?; + let bindings = core_key_bindings(selectors, resolved_keys)?; + datafog_core::transform_with_resolved_keys(&text, &findings, &config, bindings) + .map_err(js_privacy_error) + .and_then(js_transform_result) +} + +#[napi(strict, catch_unwind)] +pub fn prepare_scan_and_transform( + env: Env, + text: String, + #[napi(ts_arg_type = "ScanAndTransformConfig")] config: Unknown<'_>, +) -> napi::Result { + let config: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_scan_and_transform_config(&config).map_err(js_privacy_error)?; + let findings = datafog_core::scan_with_config(&text, config.scan_config()); + let selectors = + datafog_core::required_key_selectors(&text, &findings, config.transformation_config()) + .map_err(js_privacy_error)?; + Ok(PreparedScanAndTransform { + findings: findings + .into_iter() + .map(js_finding) + .collect::>>()?, + selectors: js_key_selectors(&selectors)?, + }) +} diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index f067f55..b01a476 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -14,4 +14,5 @@ crate-type = ["cdylib"] [dependencies] datafog-core = { path = "../../crates/core" } pyo3 = { version = "0.29", features = ["abi3-py310"] } +pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } serde_json = "1" diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index f8835f7..c9d3855 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,12 +1,13 @@ use ::datafog_core as core; use pyo3::create_exception; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyBool, PyDict, PyList, PyTuple}; create_exception!(datafog_core, DataFogConfigurationError, PyValueError); create_exception!(datafog_core, DataFogFindingError, PyValueError); create_exception!(datafog_core, DataFogInternalError, PyRuntimeError); +create_exception!(datafog_core, DataFogKeyProviderError, PyRuntimeError); /// A zero-based, end-exclusive text range. #[pyclass(frozen, skip_from_py_object)] @@ -167,7 +168,22 @@ impl Finding { #[derive(Clone, PartialEq)] struct Transformation { #[pyo3(get)] - finding: Finding, + entity_type: String, + + #[pyo3(get)] + source_byte_range: TextRange, + + #[pyo3(get)] + source_codepoint_range: TextRange, + + #[pyo3(get)] + confidence: Option, + + #[pyo3(get)] + detector_name: String, + + #[pyo3(get)] + detector_version: Option, #[pyo3(get)] strategy: String, @@ -180,20 +196,34 @@ struct Transformation { #[pyo3(get)] output_codepoint_range: TextRange, + + #[pyo3(get)] + key_ref: Option, + + #[pyo3(get)] + resolved_key_version: Option, } impl From for Transformation { fn from(transformation: core::Transformation) -> Self { Self { - finding: transformation.finding.into(), + entity_type: transformation.entity_type, + source_byte_range: transformation.source_byte_range.into(), + source_codepoint_range: transformation.source_codepoint_range.into(), + confidence: transformation.confidence, + detector_name: transformation.detector_name, + detector_version: transformation.detector_version, strategy: match transformation.strategy { core::TransformationStrategy::Redact => "redact".to_owned(), core::TransformationStrategy::Remove => "remove".to_owned(), core::TransformationStrategy::Mask(_) => "mask".to_owned(), + core::TransformationStrategy::Pseudonymize(_) => "pseudonymize".to_owned(), }, replacement: transformation.replacement, output_byte_range: transformation.output_byte_range.into(), output_codepoint_range: transformation.output_codepoint_range.into(), + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, } } } @@ -201,17 +231,18 @@ impl From for Transformation { #[pymethods] impl Transformation { fn __eq__(&self, other: PyRef<'_, Transformation>) -> bool { - self.finding.entity_type == other.finding.entity_type - && self.finding.matched_text == other.finding.matched_text - && self.finding.byte_range == other.finding.byte_range - && self.finding.codepoint_range == other.finding.codepoint_range - && self.finding.confidence == other.finding.confidence - && self.finding.detector_name == other.finding.detector_name - && self.finding.detector_version == other.finding.detector_version + self.entity_type == other.entity_type + && self.source_byte_range == other.source_byte_range + && self.source_codepoint_range == other.source_codepoint_range + && self.confidence == other.confidence + && self.detector_name == other.detector_name + && self.detector_version == other.detector_version && self.strategy == other.strategy && self.replacement == other.replacement && self.output_byte_range == other.output_byte_range && self.output_codepoint_range == other.output_codepoint_range + && self.key_ref == other.key_ref + && self.resolved_key_version == other.resolved_key_version } } @@ -246,6 +277,139 @@ impl TransformResult { } } +struct PythonKeyProvider { + provider: Py, +} + +fn provider_error_kind(error: &PyErr) -> core::KeyProviderErrorKind { + Python::attach(|py| { + let code = error + .value(py) + .getattr("code") + .and_then(|value| value.extract::()) + .ok(); + match code.as_deref() { + Some("key_not_found") => core::KeyProviderErrorKind::NotFound, + Some("key_access_denied") => core::KeyProviderErrorKind::AccessDenied, + Some("key_provider_unavailable") => core::KeyProviderErrorKind::Unavailable, + _ => core::KeyProviderErrorKind::ProviderError, + } + }) +} + +fn provider_field<'py>( + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + if let Ok(dictionary) = value.cast::() { + dictionary.get_item(name) + } else { + value.getattr(name).map(Some) + } +} + +fn resolved_key_from_python(value: &Bound<'_, PyAny>) -> core::ResolvedKey { + let key = provider_field(value, "key") + .ok() + .flatten() + .and_then(|value| value.extract::>().ok()) + .unwrap_or_default(); + let resolved_version = provider_field(value, "resolved_version") + .ok() + .flatten() + .and_then(|value| value.extract::().ok()) + .unwrap_or_default(); + core::ResolvedKey::new(key, resolved_version) +} + +impl core::KeyProvider for PythonKeyProvider { + fn resolve_key(&self, selector: core::KeySelector) -> core::KeyProviderFuture<'_> { + let provider = Python::attach(|py| self.provider.clone_ref(py)); + Box::pin(async move { + let future = Python::attach(|py| { + let awaitable = provider + .bind(py) + .call_method1("resolve_key", (selector.key_ref(), selector.key_version()))?; + pyo3_async_runtimes::tokio::into_future(awaitable) + }) + .map_err(|error| core::KeyProviderError::new(provider_error_kind(&error)))?; + let response = future + .await + .map_err(|error| core::KeyProviderError::new(provider_error_kind(&error)))?; + Ok(Python::attach(|py| { + resolved_key_from_python(response.bind(py)) + })) + }) + } +} + +/// Provider-backed asynchronous privacy manager. +#[pyclass(frozen, skip_from_py_object)] +struct PrivacyManager { + provider: Py, +} + +#[pymethods] +impl PrivacyManager { + #[new] + fn new(py: Python<'_>, provider: Py) -> PyResult { + let resolve_key = provider.bind(py).getattr("resolve_key").map_err(|_| { + PyErr::new::("provider must define resolve_key(key_ref, key_version)") + })?; + if !resolve_key.is_callable() { + return Err(PyErr::new::( + "provider resolve_key attribute must be callable", + )); + } + Ok(Self { provider }) + } + + fn transform<'py>( + &self, + py: Python<'py>, + text: String, + findings: Vec>, + config: Py, + ) -> PyResult> { + let config_value = py_to_json(py, config.bind(py), "")?; + let config = core::parse_transformation_config(&config_value) + .map_err(|error| privacy_error(py, error))?; + let findings = findings + .iter() + .map(|finding| finding.bind(py).borrow().to_core()) + .collect::>(); + let provider = self.provider.clone_ref(py); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { provider }); + let result = manager + .transform(&text, &findings, &config) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, TransformResult::from(result))) + }) + } + + fn scan_and_transform<'py>( + &self, + py: Python<'py>, + text: String, + config: Py, + ) -> PyResult> { + let config_value = py_to_json(py, config.bind(py), "")?; + let config = core::parse_scan_and_transform_config(&config_value) + .map_err(|error| privacy_error(py, error))?; + let provider = self.provider.clone_ref(py); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { provider }); + let result = manager + .scan_and_transform(&text, &config) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, TransformResult::from(result))) + }) + } +} + fn configuration_conversion_error(py: Python<'_>, path: &str, message: &str) -> PyErr { let exception = PyErr::new::(message.to_owned()); let value = exception.value(py); @@ -331,6 +495,15 @@ fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr { core::PrivacyErrorCode::InvalidFinding => { PyErr::new::(error.to_string()) } + core::PrivacyErrorCode::KeyProviderRequired + | core::PrivacyErrorCode::KeyNotFound + | core::PrivacyErrorCode::KeyAccessDenied + | core::PrivacyErrorCode::KeyProviderUnavailable + | core::PrivacyErrorCode::InvalidKeyMaterial + | core::PrivacyErrorCode::KeyProviderError + | core::PrivacyErrorCode::UnsupportedStrategy => { + PyErr::new::(error.to_string()) + } core::PrivacyErrorCode::InternalError => { PyErr::new::(error.to_string()) } @@ -414,10 +587,15 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { "DataFogInternalError", module.py().get_type::(), )?; + module.add( + "DataFogKeyProviderError", + module.py().get_type::(), + )?; module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(scan, module)?)?; module.add_function(wrap_pyfunction!(transform, module)?)?; module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?; diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index c2b40fe..74a0835 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -3,12 +3,15 @@ from __future__ import annotations import json +import asyncio from pathlib import Path from datafog_core import ( DataFogConfigurationError, DataFogFindingError, + DataFogKeyProviderError, Finding, + PrivacyManager, TextRange, scan, scan_and_transform, @@ -80,6 +83,10 @@ def main() -> None: assert explicit.text == "👋 [EMAIL] and [EMAIL]" assert len(explicit.transformations) == 2 first = explicit.transformations[0] + assert not hasattr(first, "finding") + assert not hasattr(first, "matched_text") + assert first.entity_type == "EMAIL" + assert first.detector_name.startswith("datafog-core/") assert first.replacement == "[EMAIL]" assert (first.output_byte_range.start, first.output_byte_range.end) == (5, 12) assert (first.output_codepoint_range.start, first.output_codepoint_range.end) == (2, 9) @@ -207,6 +214,59 @@ def main() -> None: assert selected.text == "Email support@example.com or call **********0100" assert len(selected.transformations) == 1 + pseudonym_config = { + "default": { + "strategy": "pseudonymize", + "key_ref": "customers/email", + "key_version": "7", + } + } + try: + transform(text, scan(text), pseudonym_config) + except DataFogKeyProviderError as error: + assert error.code == "key_provider_required" + assert error.path == "/default/key_ref" + else: + raise AssertionError("providerless pseudonymization was accepted") + + class Provider: + def __init__(self, key: bytes = bytes(range(32))) -> None: + self.key = key + self.calls: list[tuple[str, str | None]] = [] + + async def resolve_key( + self, key_ref: str, key_version: str | None + ) -> dict[str, object]: + self.calls.append((key_ref, key_version)) + return {"key": self.key, "resolved_version": "7"} + + provider = Provider() + async def provider_transform(active_provider: Provider): + return await PrivacyManager(active_provider).scan_and_transform( + "jane@example.com jane@example.com", + {"transform": pseudonym_config}, + ) + + pseudonymized = asyncio.run(provider_transform(provider)) + expected_token = "lIdYiXR1nTA9XURAF5GmA62F/aknbUP3Q2B31wnZ2hA=" + assert pseudonymized.text == f"{expected_token} {expected_token}" + assert provider.calls == [("customers/email", "7")] + for record in pseudonymized.transformations: + assert record.strategy == "pseudonymize" + assert record.replacement == expected_token + assert record.key_ref == "customers/email" + assert record.resolved_key_version == "7" + assert not hasattr(record, "finding") + assert not hasattr(record, "matched_text") + + try: + asyncio.run(provider_transform(Provider(b"short"))) + except DataFogKeyProviderError as error: + assert error.code == "invalid_key_material" + assert error.path == "/transform/default/key_ref" + else: + raise AssertionError("invalid provider key material was accepted") + try: transform( text, diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index e0a2410..4fc38a8 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -45,6 +45,13 @@ export interface ScanAndTransformConfig { export type DataFogErrorCode = | "invalid_configuration" | "invalid_finding" + | "key_provider_required" + | "key_not_found" + | "key_access_denied" + | "key_provider_unavailable" + | "invalid_key_material" + | "key_provider_error" + | "unsupported_strategy" | "internal_error"; export declare class DataFogError extends Error { @@ -70,11 +77,18 @@ export interface Finding { } export interface Transformation { - readonly finding: Finding; + readonly entityType: EntityType; + readonly sourceByteRange: TextRange; + readonly sourceCodepointRange: TextRange; + readonly confidence?: number; + readonly detectorName: string; + readonly detectorVersion?: string; readonly strategy: TransformationStrategy; readonly replacement: string; readonly outputByteRange: TextRange; readonly outputCodepointRange: TextRange; + readonly keyRef?: string; + readonly resolvedKeyVersion?: string; } export interface TransformResult { diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index e62217f..d088fcc 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -51,11 +51,18 @@ impl From for datafog_core::Finding { #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct Transformation { - finding: Finding, + entity_type: String, + source_byte_range: TextRange, + source_codepoint_range: TextRange, + confidence: Option, + detector_name: String, + detector_version: Option, strategy: &'static str, replacement: String, output_byte_range: TextRange, output_codepoint_range: TextRange, + key_ref: Option, + resolved_key_version: Option, } #[derive(Serialize)] @@ -100,15 +107,23 @@ fn result_to_js(result: datafog_core::TransformResult) -> Result "redact", datafog_core::TransformationStrategy::Remove => "remove", datafog_core::TransformationStrategy::Mask(_) => "mask", + datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", }, replacement: transformation.replacement, output_byte_range: transformation.output_byte_range.into(), output_codepoint_range: transformation.output_codepoint_range.into(), + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, }) .collect(), }; @@ -142,6 +157,15 @@ pub fn transform(text: &str, findings: JsValue, config: JsValue) -> Result>(); let config = config_value(config)?; let config = datafog_core::parse_transformation_config(&config).map_err(privacy_error)?; + if let Some(selector) = datafog_core::required_key_selectors(text, &findings, &config) + .map_err(privacy_error)? + .into_iter() + .next() + { + return Err(privacy_error( + datafog_core::PrivacyError::unsupported_strategy(selector.path()), + )); + } let result = datafog_core::transform(text, &findings, &config).map_err(privacy_error)?; result_to_js(result) } @@ -150,6 +174,20 @@ pub fn transform(text: &str, findings: JsValue, config: JsValue) -> Result Result { let config = config_value(config)?; let config = datafog_core::parse_scan_and_transform_config(&config).map_err(privacy_error)?; + let findings = datafog_core::scan_with_config(text, config.scan_config()); + if let Some(selector) = + datafog_core::required_key_selectors(text, &findings, config.transformation_config()) + .map_err(privacy_error)? + .into_iter() + .next() + { + return Err(privacy_error( + datafog_core::PrivacyError::unsupported_strategy(format!( + "/transform{}", + selector.path() + )), + )); + } let result = datafog_core::scan_and_transform(text, &config).map_err(privacy_error)?; result_to_js(result) } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4120926..b0cab75 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -12,6 +12,13 @@ keywords = ["pii", "privacy", "detection"] categories = ["text-processing"] [dependencies] +base64 = "0.22" +hmac = "0.12" regex = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" +zeroize = "1" + +[dev-dependencies] +futures = "0.3" diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index fd66dae..ff3af29 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,8 +1,14 @@ //! Core PII scanning API for DataFog. +use base64::Engine; +use hmac::{Hmac, Mac}; use regex::{Regex, RegexSet, RegexSetBuilder}; +use sha2::Sha256; use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; use std::net::{Ipv4Addr, Ipv6Addr}; +use std::pin::Pin; use std::sync::LazyLock; +use zeroize::Zeroize; /// A zero-based, end-exclusive range in the coordinate system named by its /// containing field. @@ -34,7 +40,7 @@ pub struct Finding { } /// A privacy transformation applied to a finding. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum TransformationStrategy { /// Replace the finding with its unnumbered entity-type placeholder. Redact, @@ -42,6 +48,68 @@ pub enum TransformationStrategy { Remove, /// Replace non-revealed code points with a configured character. Mask(MaskConfig), + /// Replace the exact finding value with a deterministic keyed pseudonym. + Pseudonymize(PseudonymizeConfig), +} + +/// Key selector for deterministic one-way pseudonymization. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct PseudonymizeConfig { + key_ref: String, + key_version: Option, +} + +/// Reason a pseudonymization configuration is invalid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PseudonymizeConfigError { + /// The key reference is empty or contains only whitespace. + EmptyKeyRef, + /// The supplied key version is empty or contains only whitespace. + EmptyKeyVersion, +} + +impl std::fmt::Display for PseudonymizeConfigError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::EmptyKeyRef => formatter.write_str("key reference must not be empty"), + Self::EmptyKeyVersion => formatter.write_str("key version must not be empty"), + } + } +} + +impl std::error::Error for PseudonymizeConfigError {} + +impl PseudonymizeConfig { + /// Create a validated pseudonymization key selector. + pub fn new( + key_ref: impl Into, + key_version: Option, + ) -> Result { + let key_ref = key_ref.into(); + if key_ref.trim().is_empty() { + return Err(PseudonymizeConfigError::EmptyKeyRef); + } + if key_version + .as_ref() + .is_some_and(|version| version.trim().is_empty()) + { + return Err(PseudonymizeConfigError::EmptyKeyVersion); + } + Ok(Self { + key_ref, + key_version, + }) + } + + /// Provider-specific key reference. + pub fn key_ref(&self) -> &str { + &self.key_ref + } + + /// Requested key version or alias, when supplied. + pub fn key_version(&self) -> Option<&str> { + self.key_version.as_deref() + } } const MAX_REGEX_RULES: usize = 100; @@ -241,11 +309,21 @@ impl TransformationConfig { }) } - fn strategy_for(&self, finding: &Finding) -> TransformationStrategy { + fn strategy_for(&self, finding: &Finding) -> &TransformationStrategy { self.overrides .get(&finding.entity_type) - .copied() - .unwrap_or(self.default) + .unwrap_or(&self.default) + } + + fn strategy_path_for(&self, finding: &Finding) -> String { + if self.overrides.contains_key(&finding.entity_type) { + format!( + "/overrides/{}/key_ref", + json_pointer_segment(&finding.entity_type) + ) + } else { + "/default/key_ref".to_owned() + } } fn compile_regex_allowlists(&mut self) -> Result<(), PrivacyError> { @@ -655,10 +733,48 @@ fn parse_strategy_config( ) }) } + "pseudonymize" => { + reject_unknown_fields(object, &["strategy", "key_ref", "key_version"], path)?; + let key_ref_path = format!("{path}/key_ref"); + let key_ref = object.get("key_ref").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + &key_ref_path, + "pseudonymization requires key_ref", + ) + })?; + let key_ref = require_string(key_ref, &key_ref_path, "key_ref must be a string")?; + let key_version = object + .get("key_version") + .map(|value| { + require_string( + value, + &format!("{path}/key_version"), + "key_version must be a string", + ) + }) + .transpose()?; + PseudonymizeConfig::new(key_ref, key_version) + .map(TransformationStrategy::Pseudonymize) + .map_err(|error| match error { + PseudonymizeConfigError::EmptyKeyRef => PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + key_ref_path, + "key_ref must not be empty or whitespace-only", + ), + PseudonymizeConfigError::EmptyKeyVersion => { + PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + format!("{path}/key_version"), + "key_version must not be empty or whitespace-only", + ) + } + }) + } _ => Err(PrivacyError::invalid_configuration( PrivacyErrorReason::InvalidValue, strategy_path, - "strategy must be redact, mask, or remove", + "strategy must be redact, mask, remove, or pseudonymize", )), } } @@ -776,11 +892,151 @@ impl Default for MaskConfig { } } +/// One provider key requested by a prepared transformation. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct KeySelector { + config: PseudonymizeConfig, + path: String, +} + +impl KeySelector { + /// Provider-specific key reference. + pub fn key_ref(&self) -> &str { + self.config.key_ref() + } + + /// Requested key version or alias, when supplied. + pub fn key_version(&self) -> Option<&str> { + self.config.key_version() + } + + /// Configuration path used for sanitized provider errors. + pub fn path(&self) -> &str { + &self.path + } +} + +/// Resolved provider response containing short-lived secret key material. +pub struct ResolvedKey { + key: Vec, + resolved_version: String, +} + +impl ResolvedKey { + /// Create a provider response. The manager validates its contents before + /// any transformation is applied. + pub fn new(key: Vec, resolved_version: impl Into) -> Self { + Self { + key, + resolved_version: resolved_version.into(), + } + } + + fn key(&self) -> &[u8] { + &self.key + } + + /// Concrete provider version used for this response. + pub fn resolved_version(&self) -> &str { + &self.resolved_version + } +} + +impl std::fmt::Debug for ResolvedKey { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ResolvedKey") + .field("key", &"[REDACTED]") + .field("resolved_version", &self.resolved_version) + .finish() + } +} + +impl Drop for ResolvedKey { + fn drop(&mut self) { + self.key.zeroize(); + } +} + +/// Provider failure category independent of any cloud SDK. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyProviderErrorKind { + NotFound, + AccessDenied, + Unavailable, + ProviderError, +} + +/// Sanitized failure returned by a key provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyProviderError { + kind: KeyProviderErrorKind, +} + +impl KeyProviderError { + /// Create a sanitized provider failure. + pub fn new(kind: KeyProviderErrorKind) -> Self { + Self { kind } + } + + /// Stable provider failure category. + pub fn kind(self) -> KeyProviderErrorKind { + self.kind + } +} + +impl std::fmt::Display for KeyProviderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("key provider could not resolve the requested key") + } +} + +impl std::error::Error for KeyProviderError {} + +/// Future returned by a vendor-neutral asynchronous key provider. +pub type KeyProviderFuture<'a> = + Pin> + Send + 'a>>; + +/// Runtime boundary for resolving pseudonymization key references. +pub trait KeyProvider: Send + Sync { + /// Resolve one key selector. Provider implementations own retries, + /// timeouts, authentication, decoding, and optional caching. + fn resolve_key(&self, selector: KeySelector) -> KeyProviderFuture<'_>; +} + +/// Provider-backed privacy operation manager. +#[derive(Debug)] +pub struct PrivacyManager

{ + provider: P, +} + +impl

PrivacyManager

{ + /// Create a manager with one runtime provider. + pub fn new(provider: P) -> Self { + Self { provider } + } + + /// Borrow the configured provider. + pub fn provider(&self) -> &P { + &self.provider + } +} + /// One transformation applied to the source text. #[derive(Debug, Clone, PartialEq)] pub struct Transformation { - /// The source finding that was transformed. - pub finding: Finding, + /// Canonical PII type of the source finding. + pub entity_type: String, + /// Source range in UTF-8 bytes. + pub source_byte_range: TextRange, + /// Source range in Unicode code points. + pub source_codepoint_range: TextRange, + /// Detector confidence, when available. + pub confidence: Option, + /// Stable detector name. + pub detector_name: String, + /// Detector implementation version, when available. + pub detector_version: Option, /// Strategy applied to the finding. pub strategy: TransformationStrategy, /// Exact replacement inserted into the output text. @@ -789,6 +1045,10 @@ pub struct Transformation { pub output_byte_range: TextRange, /// Range of the replacement in Unicode code points in the output text. pub output_codepoint_range: TextRange, + /// Provider-specific key reference for pseudonymization only. + pub key_ref: Option, + /// Concrete provider version for pseudonymization only. + pub resolved_key_version: Option, } /// Text and audit records produced by a transformation. @@ -828,6 +1088,20 @@ pub enum PrivacyErrorCode { InvalidConfiguration, /// One caller-supplied finding is invalid. InvalidFinding, + /// Pseudonymization was selected without a runtime provider. + KeyProviderRequired, + /// The requested provider key does not exist. + KeyNotFound, + /// The provider denied access to the requested key. + KeyAccessDenied, + /// The provider is temporarily unavailable. + KeyProviderUnavailable, + /// The provider returned malformed or weak key material. + InvalidKeyMaterial, + /// The provider failed without a more specific safe category. + KeyProviderError, + /// The selected runtime intentionally cannot execute this strategy. + UnsupportedStrategy, /// An unexpected non-caller-correctable failure occurred. InternalError, } @@ -838,6 +1112,13 @@ impl PrivacyErrorCode { match self { Self::InvalidConfiguration => "invalid_configuration", Self::InvalidFinding => "invalid_finding", + Self::KeyProviderRequired => "key_provider_required", + Self::KeyNotFound => "key_not_found", + Self::KeyAccessDenied => "key_access_denied", + Self::KeyProviderUnavailable => "key_provider_unavailable", + Self::InvalidKeyMaterial => "invalid_key_material", + Self::KeyProviderError => "key_provider_error", + Self::UnsupportedStrategy => "unsupported_strategy", Self::InternalError => "internal_error", } } @@ -959,6 +1240,74 @@ impl PrivacyError { } } + fn key_error(code: PrivacyErrorCode, path: impl Into, message: &'static str) -> Self { + Self { + code, + reason: None, + path: Some(path.into()), + finding_index: None, + message: message.to_owned(), + } + } + + fn provider_required(path: impl Into) -> Self { + Self::key_error( + PrivacyErrorCode::KeyProviderRequired, + path, + "pseudonymization requires a runtime key provider", + ) + } + + fn invalid_key_material(path: impl Into) -> Self { + Self::key_error( + PrivacyErrorCode::InvalidKeyMaterial, + path, + "key provider returned invalid key material", + ) + } + + fn internal(message: &'static str) -> Self { + Self { + code: PrivacyErrorCode::InternalError, + reason: None, + path: None, + finding_index: None, + message: message.to_owned(), + } + } + + fn from_provider_error(path: impl Into, error: KeyProviderError) -> Self { + let (code, message) = match error.kind() { + KeyProviderErrorKind::NotFound => ( + PrivacyErrorCode::KeyNotFound, + "key provider could not find the requested key", + ), + KeyProviderErrorKind::AccessDenied => ( + PrivacyErrorCode::KeyAccessDenied, + "key provider denied access to the requested key", + ), + KeyProviderErrorKind::Unavailable => ( + PrivacyErrorCode::KeyProviderUnavailable, + "key provider is temporarily unavailable", + ), + KeyProviderErrorKind::ProviderError => ( + PrivacyErrorCode::KeyProviderError, + "key provider could not resolve the requested key", + ), + }; + Self::key_error(code, path, message) + } + + /// Create a structured unsupported-strategy error for a binding that + /// intentionally cannot execute a canonical strategy. + pub fn unsupported_strategy(path: impl Into) -> Self { + Self::key_error( + PrivacyErrorCode::UnsupportedStrategy, + path, + "the selected runtime does not support pseudonymization", + ) + } + fn prefixed(mut self, prefix: &str) -> Self { if let Some(path) = &mut self.path { *path = format!("{prefix}{path}"); @@ -1064,6 +1413,66 @@ pub fn transform( findings: &[Finding], config: &TransformationConfig, ) -> Result { + let selected_findings = select_findings(text, findings, config)?; + if let Some(selector) = key_selectors(config, &selected_findings).into_iter().next() { + return Err(PrivacyError::provider_required(selector.path)); + } + apply_transformations(text, &selected_findings, config, &BTreeMap::new()) +} + +/// Return the distinct provider keys required after validation, filtering, +/// allowlists, duplicate handling, and overlap resolution. +pub fn required_key_selectors( + text: &str, + findings: &[Finding], + config: &TransformationConfig, +) -> Result, PrivacyError> { + let selected_findings = select_findings(text, findings, config)?; + Ok(key_selectors(config, &selected_findings)) +} + +/// One resolved key associated with the selector that requested it. +pub struct ResolvedKeyBinding { + selector: KeySelector, + key: ResolvedKey, +} + +impl ResolvedKeyBinding { + /// Associate one provider response with its original selector. + pub fn new(selector: KeySelector, key: ResolvedKey) -> Self { + Self { selector, key } + } +} + +impl std::fmt::Debug for ResolvedKeyBinding { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ResolvedKeyBinding") + .field("selector", &self.selector) + .field("key", &self.key) + .finish() + } +} + +/// Apply a transformation using provider responses already resolved by a thin +/// language binding or another trusted orchestration layer. +pub fn transform_with_resolved_keys( + text: &str, + findings: &[Finding], + config: &TransformationConfig, + resolved_keys: Vec, +) -> Result { + let selected_findings = select_findings(text, findings, config)?; + let selectors = key_selectors(config, &selected_findings); + let resolved_keys = validate_resolved_keys(selectors, resolved_keys)?; + apply_transformations(text, &selected_findings, config, &resolved_keys) +} + +fn select_findings( + text: &str, + findings: &[Finding], + config: &TransformationConfig, +) -> Result, PrivacyError> { for (finding_index, finding) in findings.iter().enumerate() { if let Err(kind) = validate_finding(text, finding) { return Err(PrivacyError::invalid_finding(finding_index, kind)); @@ -1093,17 +1502,77 @@ pub fn transform( finding.entity_type.clone(), ) }); - let selected_findings = resolve_overlaps(selected_findings); + Ok(resolve_overlaps(selected_findings)) +} + +fn key_selectors(config: &TransformationConfig, selected_findings: &[Finding]) -> Vec { + let mut selectors = BTreeMap::new(); + for finding in selected_findings { + if let TransformationStrategy::Pseudonymize(pseudonymize) = config.strategy_for(finding) { + selectors + .entry(pseudonymize.clone()) + .or_insert_with(|| KeySelector { + config: pseudonymize.clone(), + path: config.strategy_path_for(finding), + }); + } + } + selectors.into_values().collect() +} + +fn validate_resolved_key(selector: &KeySelector, key: &ResolvedKey) -> Result<(), PrivacyError> { + if key.key().len() != 32 || key.resolved_version().trim().is_empty() { + return Err(PrivacyError::invalid_key_material(selector.path.clone())); + } + Ok(()) +} +fn validate_resolved_keys( + selectors: Vec, + bindings: Vec, +) -> Result, PrivacyError> { + let expected = selectors + .iter() + .map(|selector| (selector.config.clone(), selector)) + .collect::>(); + let mut resolved = BTreeMap::new(); + for binding in bindings { + let Some(selector) = expected.get(&binding.selector.config) else { + return Err(PrivacyError::invalid_key_material(binding.selector.path)); + }; + validate_resolved_key(selector, &binding.key)?; + if resolved + .insert(binding.selector.config, binding.key) + .is_some() + { + return Err(PrivacyError::invalid_key_material(selector.path.clone())); + } + } + for selector in selectors { + if !resolved.contains_key(&selector.config) { + return Err(PrivacyError::provider_required(selector.path)); + } + } + Ok(resolved) +} + +fn apply_transformations( + text: &str, + selected_findings: &[Finding], + config: &TransformationConfig, + resolved_keys: &BTreeMap, +) -> Result { let mut output = String::with_capacity(text.len()); let mut transformations = Vec::with_capacity(selected_findings.len()); let mut source_byte_cursor = 0; - for finding in &selected_findings { + for finding in selected_findings { output.push_str(&text[source_byte_cursor..finding.byte_range.start]); let output_byte_start = output.len(); let output_codepoint_start = output.chars().count(); let strategy = config.strategy_for(finding); + let mut key_ref = None; + let mut resolved_key_version = None; let replacement = match strategy { TransformationStrategy::Redact => format!("[{}]", finding.entity_type), TransformationStrategy::Remove => String::new(), @@ -1125,12 +1594,29 @@ pub fn transform( }) .collect() } + TransformationStrategy::Pseudonymize(pseudonymize) => { + let selector_path = config.strategy_path_for(finding); + let resolved = resolved_keys + .get(pseudonymize) + .ok_or_else(|| PrivacyError::provider_required(selector_path))?; + let mut mac = Hmac::::new_from_slice(resolved.key()) + .map_err(|_| PrivacyError::internal("could not initialize HMAC-SHA-256"))?; + mac.update(finding.matched_text.as_bytes()); + key_ref = Some(pseudonymize.key_ref.clone()); + resolved_key_version = Some(resolved.resolved_version.clone()); + base64::engine::general_purpose::STANDARD.encode(mac.finalize().into_bytes()) + } }; output.push_str(&replacement); transformations.push(Transformation { - finding: finding.clone(), - strategy, + entity_type: finding.entity_type.clone(), + source_byte_range: finding.byte_range, + source_codepoint_range: finding.codepoint_range, + confidence: finding.confidence, + detector_name: finding.detector_name.clone(), + detector_version: finding.detector_version.clone(), + strategy: strategy.clone(), replacement, output_byte_range: TextRange { start: output_byte_start, @@ -1140,6 +1626,8 @@ pub fn transform( start: output_codepoint_start, end: output.chars().count(), }, + key_ref, + resolved_key_version, }); source_byte_cursor = finding.byte_range.end; } @@ -1151,6 +1639,46 @@ pub fn transform( }) } +impl PrivacyManager

{ + /// Transform caller-supplied findings after atomically resolving every + /// distinct key selected by the request. + pub async fn transform( + &self, + text: &str, + findings: &[Finding], + config: &TransformationConfig, + ) -> Result { + let selected_findings = select_findings(text, findings, config)?; + let selectors = key_selectors(config, &selected_findings); + let mut resolved = BTreeMap::new(); + for selector in selectors { + let key = self + .provider + .resolve_key(selector.clone()) + .await + .map_err(|error| PrivacyError::from_provider_error(selector.path.clone(), error))?; + validate_resolved_key(&selector, &key)?; + resolved.insert(selector.config, key); + } + apply_transformations(text, &selected_findings, config, &resolved) + } + + /// Scan and transform after atomically resolving every selected key. + pub async fn scan_and_transform( + &self, + text: &str, + config: &ScanAndTransformConfig, + ) -> Result { + self.transform( + text, + &scan_with_config(text, config.scan_config()), + config.transformation_config(), + ) + .await + .map_err(|error| error.prefixed("/transform")) + } +} + /// Scan text and transform the resulting findings in one explicit convenience operation. pub fn scan_and_transform( text: &str, @@ -1161,6 +1689,7 @@ pub fn scan_and_transform( &scan_with_config(text, config.scan_config()), config.transformation_config(), ) + .map_err(|error| error.prefixed("/transform")) } fn findings_are_duplicates(left: &Finding, right: &Finding) -> bool { @@ -1941,18 +2470,88 @@ fn detect_ip_address(text: &str, candidates: &mut Vec) { #[cfg(test)] mod tests { use super::{ - Finding, FindingValidationError, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES, MaskConfig, + Finding, FindingValidationError, KeyProvider, KeyProviderError, KeyProviderErrorKind, + KeyProviderFuture, KeySelector, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES, MaskConfig, MaskConfigError, MaskReveal, PrivacyError, PrivacyErrorCode, PrivacyErrorReason, - RegexAllowRule, ScanAndTransformConfig, TextRange, TransformationConfig, - TransformationStrategy, parse_scan_and_transform_config, parse_transformation_config, scan, - scan_and_transform, transform, + PrivacyManager, RegexAllowRule, ResolvedKey, ScanAndTransformConfig, TextRange, + TransformationConfig, TransformationStrategy, parse_scan_and_transform_config, + parse_transformation_config, scan, scan_and_transform, transform, }; + use futures::executor::block_on; use serde_json::json; + use std::collections::BTreeMap; + use std::sync::Mutex; + + #[derive(Default)] + struct TestKeyProvider { + responses: BTreeMap<(String, Option), (Vec, String)>, + failure: Option, + calls: Mutex)>>, + } + + impl TestKeyProvider { + fn with_key( + mut self, + key_ref: &str, + requested_version: Option<&str>, + key: Vec, + resolved_version: &str, + ) -> Self { + self.responses.insert( + (key_ref.to_owned(), requested_version.map(str::to_owned)), + (key, resolved_version.to_owned()), + ); + self + } + + fn failing(kind: KeyProviderErrorKind) -> Self { + Self { + failure: Some(kind), + ..Self::default() + } + } + + fn call_count(&self) -> usize { + self.calls.lock().unwrap().len() + } + } + + impl KeyProvider for TestKeyProvider { + fn resolve_key(&self, selector: KeySelector) -> KeyProviderFuture<'_> { + let identity = ( + selector.key_ref().to_owned(), + selector.key_version().map(str::to_owned), + ); + self.calls.lock().unwrap().push(identity.clone()); + let response = self.responses.get(&identity).cloned(); + let failure = self.failure; + Box::pin(async move { + if let Some(kind) = failure { + return Err(KeyProviderError::new(kind)); + } + let (key, version) = response + .ok_or_else(|| KeyProviderError::new(KeyProviderErrorKind::NotFound))?; + Ok(ResolvedKey::new(key, version)) + }) + } + } fn config(strategy: TransformationStrategy) -> TransformationConfig { TransformationConfig::new(strategy) } + fn assert_transformation_source(transformation: &super::Transformation, finding: &Finding) { + assert_eq!(transformation.entity_type, finding.entity_type); + assert_eq!(transformation.source_byte_range, finding.byte_range); + assert_eq!( + transformation.source_codepoint_range, + finding.codepoint_range + ); + assert_eq!(transformation.confidence, finding.confidence); + assert_eq!(transformation.detector_name, finding.detector_name); + assert_eq!(transformation.detector_version, finding.detector_version); + } + fn expected_finding( entity_type: &str, matched_text: &str, @@ -2237,7 +2836,7 @@ mod tests { assert_eq!(result.text, "Contact [EMAIL]"); assert_eq!(result.transformations.len(), 1); let transformation = &result.transformations[0]; - assert_eq!(transformation.finding, findings[0]); + assert_transformation_source(transformation, &findings[0]); assert_eq!(transformation.strategy, TransformationStrategy::Redact); assert_eq!(transformation.replacement, "[EMAIL]"); assert_eq!( @@ -2250,6 +2849,223 @@ mod tests { ); } + #[test] + fn parses_pseudonymization_and_requires_a_runtime_provider() { + let config = parse_transformation_config(&json!({ + "default": { + "strategy": "pseudonymize", + "key_ref": "customers/email", + "key_version": "7" + } + })) + .unwrap(); + let text = "Email jane@example.com"; + let error = transform(text, &scan(text), &config).unwrap_err(); + + assert_eq!(error.code(), PrivacyErrorCode::KeyProviderRequired); + assert_eq!(error.path(), Some("/default/key_ref")); + + for invalid in [ + json!({"default": {"strategy": "pseudonymize"}}), + json!({"default": {"strategy": "pseudonymize", "key_ref": " "}}), + json!({ + "default": { + "strategy": "pseudonymize", + "key_ref": "customers/email", + "key_version": "" + } + }), + json!({ + "default": { + "strategy": "pseudonymize", + "key_ref": "customers/email", + "algorithm": "sha512" + } + }), + ] { + assert!(parse_transformation_config(&invalid).is_err()); + } + } + + #[test] + fn pseudonymizes_exact_utf8_with_full_padded_base64_hmac_sha256() { + let config = parse_transformation_config(&json!({ + "default": { + "strategy": "pseudonymize", + "key_ref": "customers/email", + "key_version": "7" + } + })) + .unwrap(); + let manager = PrivacyManager::new(TestKeyProvider::default().with_key( + "customers/email", + Some("7"), + (0_u8..32).collect(), + "7", + )); + + let result = block_on(manager.transform( + "Email jane@example.com", + &scan("Email jane@example.com"), + &config, + )) + .unwrap(); + + assert_eq!( + result.text, + "Email lIdYiXR1nTA9XURAF5GmA62F/aknbUP3Q2B31wnZ2hA=" + ); + let record = &result.transformations[0]; + assert_eq!(record.replacement.len(), 44); + assert_eq!(record.key_ref.as_deref(), Some("customers/email")); + assert_eq!(record.resolved_key_version.as_deref(), Some("7")); + assert_eq!(record.entity_type, "EMAIL"); + assert_eq!(record.source_byte_range, TextRange { start: 6, end: 22 }); + assert_eq!(manager.provider().call_count(), 1); + } + + #[test] + fn the_key_alone_defines_linkage_scope_across_entity_types() { + let text = "jane@example.com jane@example.com"; + let mut findings = scan(text); + findings[1].entity_type = "CUSTOM_IDENTIFIER".to_owned(); + let config = parse_transformation_config(&json!({ + "default": {"strategy": "pseudonymize", "key_ref": "shared"} + })) + .unwrap(); + let manager = PrivacyManager::new(TestKeyProvider::default().with_key( + "shared", + None, + vec![42; 32], + "2026-08-27", + )); + + let result = block_on(manager.transform(text, &findings, &config)).unwrap(); + + assert_eq!(result.transformations.len(), 2); + assert_eq!( + result.transformations[0].replacement, + result.transformations[1].replacement + ); + assert_eq!(manager.provider().call_count(), 1); + } + + #[test] + fn exact_input_and_key_material_changes_produce_different_pseudonyms() { + let config = parse_transformation_config(&json!({ + "default": {"strategy": "pseudonymize", "key_ref": "key"} + })) + .unwrap(); + let lower = "jane@example.com"; + let upper = "Jane@example.com"; + let lower_finding = supplied_ascii_finding(lower, "EMAIL", 0, lower.len(), None, "test"); + let upper_finding = supplied_ascii_finding(upper, "EMAIL", 0, upper.len(), None, "test"); + let manager_a = + PrivacyManager::new(TestKeyProvider::default().with_key("key", None, vec![1; 32], "1")); + let manager_b = + PrivacyManager::new(TestKeyProvider::default().with_key("key", None, vec![2; 32], "2")); + + let lower_a = block_on(manager_a.transform(lower, &[lower_finding], &config)).unwrap(); + let upper_a = + block_on(manager_a.transform(upper, &[upper_finding.clone()], &config)).unwrap(); + let upper_b = block_on(manager_b.transform(upper, &[upper_finding], &config)).unwrap(); + + assert_ne!( + lower_a.transformations[0].replacement, + upper_a.transformations[0].replacement + ); + assert_ne!( + upper_a.transformations[0].replacement, + upper_b.transformations[0].replacement + ); + } + + #[test] + fn multiple_selected_keys_are_deduplicated_and_resolved_before_application() { + let text = "jane@example.com jane@example.com (212) 555-0100"; + let findings = scan(text); + let config = parse_transformation_config(&json!({ + "default": {"strategy": "pseudonymize", "key_ref": "email-key"}, + "overrides": { + "PHONE": {"strategy": "pseudonymize", "key_ref": "phone-key", "key_version": "9"} + } + })) + .unwrap(); + let manager = PrivacyManager::new( + TestKeyProvider::default() + .with_key("email-key", None, vec![3; 32], "4") + .with_key("phone-key", Some("9"), vec![4; 32], "9"), + ); + + let result = block_on(manager.transform(text, &findings, &config)).unwrap(); + + assert_eq!(result.transformations.len(), 3); + assert_eq!(manager.provider().call_count(), 2); + assert_eq!( + result.transformations[0].resolved_key_version.as_deref(), + Some("4") + ); + assert_eq!( + result.transformations[1].resolved_key_version.as_deref(), + Some("4") + ); + assert_eq!( + result.transformations[2].resolved_key_version.as_deref(), + Some("9") + ); + } + + #[test] + fn invalid_or_unavailable_keys_fail_closed_without_provider_retries() { + let text = "Email jane@example.com"; + let findings = scan(text); + let config = parse_transformation_config(&json!({ + "default": {"strategy": "pseudonymize", "key_ref": "key"} + })) + .unwrap(); + let invalid_manager = + PrivacyManager::new(TestKeyProvider::default().with_key("key", None, vec![0; 31], "1")); + let unavailable_manager = + PrivacyManager::new(TestKeyProvider::failing(KeyProviderErrorKind::Unavailable)); + + let invalid = block_on(invalid_manager.transform(text, &findings, &config)).unwrap_err(); + let unavailable = + block_on(unavailable_manager.transform(text, &findings, &config)).unwrap_err(); + + assert_eq!(invalid.code(), PrivacyErrorCode::InvalidKeyMaterial); + assert_eq!(unavailable.code(), PrivacyErrorCode::KeyProviderUnavailable); + assert_eq!(unavailable_manager.provider().call_count(), 1); + } + + #[test] + fn unused_pseudonymization_keys_are_not_resolved() { + let text = "Email support@example.com"; + let findings = scan(text); + let config = parse_transformation_config(&json!({ + "default": {"strategy": "pseudonymize", "key_ref": "key"}, + "allow": {"exact": {"EMAIL": ["support@example.com"]}} + })) + .unwrap(); + let manager = PrivacyManager::new(TestKeyProvider::failing( + KeyProviderErrorKind::ProviderError, + )); + + let result = block_on(manager.transform(text, &findings, &config)).unwrap(); + + assert_eq!(result.text, text); + assert!(result.transformations.is_empty()); + assert_eq!(manager.provider().call_count(), 0); + } + + #[test] + fn resolved_key_debug_output_redacts_material() { + let key = ResolvedKey::new(vec![7; 32], "1"); + let debug = format!("{key:?}"); + + assert!(debug.contains("[REDACTED]")); + assert!(!debug.contains("7, 7")); + } + #[test] fn entity_override_replaces_the_default_strategy() { let text = "Email jane@example.com or call (212) 555-0100"; @@ -2286,7 +3102,7 @@ mod tests { assert_eq!(result.text, "Email jane@example.com or call [PHONE]"); assert_eq!(result.transformations.len(), 1); - assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + assert_eq!(result.transformations[0].entity_type, "PHONE"); } #[test] @@ -2302,7 +3118,7 @@ mod tests { assert_eq!(result.text, "[PHONE]@example.com"); assert_eq!(result.transformations.len(), 1); - assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + assert_eq!(result.transformations[0].entity_type, "PHONE"); } #[test] @@ -2394,7 +3210,7 @@ mod tests { "Email support@example.com or call **********0100" ); assert_eq!(result.transformations.len(), 1); - assert_eq!(result.transformations[0].finding.entity_type, "PHONE"); + assert_eq!(result.transformations[0].entity_type, "PHONE"); assert_eq!(convenience, result); } @@ -2570,7 +3386,7 @@ mod tests { let findings = scan(text); let strategy = TransformationStrategy::Mask(MaskConfig::default()); - let result = transform(text, &findings, &config(strategy)).unwrap(); + let result = transform(text, &findings, &config(strategy.clone())).unwrap(); assert_eq!(result.text, "Email ****************"); assert_eq!(result.transformations[0].strategy, strategy); @@ -2796,7 +3612,7 @@ mod tests { assert_eq!(result.text, "Email [EMAIL]"); assert_eq!(result.transformations.len(), 1); - assert_eq!(result.transformations[0].finding, higher_confidence); + assert_transformation_source(&result.transformations[0], &higher_confidence); } #[test] @@ -2821,7 +3637,7 @@ mod tests { assert_eq!(result.text, "[ORGANIZATION] announced"); assert_eq!(result.transformations.len(), 1); - assert_eq!(result.transformations[0].finding, outer); + assert_transformation_source(&result.transformations[0], &outer); } #[test] @@ -2864,7 +3680,7 @@ mod tests { .unwrap(); assert_eq!(result.text, "[ZETA]"); - assert_eq!(result.transformations[0].finding, higher); + assert_transformation_source(&result.transformations[0], &higher); } #[test] @@ -2881,7 +3697,7 @@ mod tests { .unwrap(); assert_eq!(result.text, "[ALPHA]"); - assert_eq!(result.transformations[0].finding, unscored); + assert_transformation_source(&result.transformations[0], &unscored); } #[test] @@ -2898,7 +3714,7 @@ mod tests { .unwrap(); assert_eq!(result.text, "[ZETA]ef"); - assert_eq!(result.transformations[0].finding, earlier); + assert_transformation_source(&result.transformations[0], &earlier); } #[test] diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 5164040..226ef63 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -142,7 +142,7 @@ keyed pseudonymization. ## Slice 6: One-way pseudonymization -**Status: contract accepted; implementation pending** +**Status: complete** - Add `pseudonymize` with required `key_ref` and optional `key_version`. - Use fixed HMAC-SHA-256 over the exact UTF-8 matched value and encode the full diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index 2d63f55..d45842a 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -30,7 +30,7 @@ function writeConsumerTest() { import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { DataFogError, scan, scanAndTransform, transform } from "@datafog/node"; +import { DataFogError, PrivacyManager, scan, scanAndTransform, transform } from "@datafog/node"; const fixturesDirectory = process.argv[2]; @@ -100,6 +100,9 @@ assert.equal(explicit.transformations.length, 2); assert.equal(explicit.transformations[0].replacement, "[EMAIL]"); assert.deepEqual(explicit.transformations[0].outputByteRange, { start: 5, end: 12 }); assert.deepEqual(explicit.transformations[0].outputCodepointRange, { start: 2, end: 9 }); +assert.equal("finding" in explicit.transformations[0], false); +assert.equal("matchedText" in explicit.transformations[0], false); +assert.equal(explicit.transformations[0].entityType, "EMAIL"); assert.equal( scanAndTransform("Email jane@example.com", { transform: { default: { strategy: "mask" } }, @@ -193,6 +196,53 @@ const selected = scanAndTransform( assert.equal(selected.text, "Email support@example.com or call **********0100"); assert.equal(selected.transformations.length, 1); +const pseudonymConfig = { + default: { + strategy: "pseudonymize", + key_ref: "customers/email", + key_version: "7", + }, +}; +assert.throws( + () => transform(transformText, scan(transformText), pseudonymConfig), + (error) => + error instanceof DataFogError && + error.code === "key_provider_required" && + error.path === "/default/key_ref", +); +const providerCalls = []; +const manager = new PrivacyManager({ + async resolveKey(request) { + providerCalls.push(request); + return { key: Uint8Array.from({ length: 32 }, (_, index) => index), resolvedVersion: "7" }; + }, +}); +const pseudonymized = await manager.scanAndTransform( + "jane@example.com jane@example.com", + { transform: pseudonymConfig }, +); +const expectedToken = "lIdYiXR1nTA9XURAF5GmA62F/aknbUP3Q2B31wnZ2hA="; +assert.equal(pseudonymized.text, expectedToken + " " + expectedToken); +assert.deepEqual(providerCalls, [{ keyRef: "customers/email", keyVersion: "7" }]); +for (const record of pseudonymized.transformations) { + assert.equal(record.replacement, expectedToken); + assert.equal(record.keyRef, "customers/email"); + assert.equal(record.resolvedKeyVersion, "7"); + assert.equal("finding" in record, false); + assert.equal("matchedText" in record, false); +} +await assert.rejects( + new PrivacyManager({ + async resolveKey() { + return { key: new Uint8Array(31), resolvedVersion: "7" }; + }, + }).scanAndTransform("Email jane@example.com", { transform: pseudonymConfig }), + (error) => + error instanceof DataFogError && + error.code === "invalid_key_material" && + error.path === "/transform/default/key_ref", +); + assert.throws( () => transform(transformText, scan(transformText), { default: { strategy: "redact" }, @@ -215,9 +265,11 @@ import { scan, scanAndTransform, transform, + PrivacyManager, type EntityType, type Finding, type MaskRevealConfig, + type KeyProvider, type ScanAndTransformConfig, type TextRange, type TransformationConfig, @@ -246,12 +298,23 @@ const maskConfig: TransformationConfig = { }; const combined: ScanAndTransformConfig = { transform: maskConfig }; const masked: TransformResult = scanAndTransform("Email jane@example.com", combined); +const provider: KeyProvider = { + async resolveKey() { + return { key: new Uint8Array(32), resolvedVersion: "1" }; + }, +}; +const pseudonymized: Promise = new PrivacyManager(provider).transform( + "Email jane@example.com", + findings, + { default: { strategy: "pseudonymize", key_ref: "customer/email" } }, +); void entityType; void range; void explicit; void convenience; void masked; +void pseudonymized; `.trimStart(), ); diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index fb79ea1..518d8f7 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -306,6 +306,12 @@ try { ) { throw new Error("transformation records do not select their replacements"); } + if ( + "finding" in explicit.transformations[0] || + "matchedText" in explicit.transformations[0] + ) { + throw new Error("transformation records must not echo original PII"); + } if ( scanAndTransform("Email jane@example.com", { @@ -425,6 +431,23 @@ try { throw new Error("selection, overrides, or allowlists failed"); } + try { + scanAndTransform("Email jane@example.com", { + transform: { + default: { strategy: "pseudonymize", key_ref: "customers/email" }, + }, + }); + throw new Error("browser pseudonymization should be unsupported"); + } catch (error) { + if ( + !(error instanceof DataFogError) || + error.code !== "unsupported_strategy" || + error.path !== "/transform/default/key_ref" + ) { + throw error; + } + } + try { transform(text, findings, { default: { strategy: "redact" },