From e9c7f492b63b665d1869571746cdd232e49b272f Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:47:49 -0700 Subject: [PATCH] Implement reversible tokenization and restoration --- README.md | 28 +- bindings/node/dts-header.d.ts | 59 +- bindings/node/index.d.ts | 107 +- bindings/node/index.js | 117 +- bindings/node/src/lib.rs | 210 ++++ bindings/python/src/lib.rs | 364 +++++- bindings/python/tests/test_installed.py | 70 ++ bindings/wasm/index.d.ts | 25 +- bindings/wasm/index.js | 13 + bindings/wasm/src/lib.rs | 43 + crates/core/src/lib.rs | 1434 ++++++++++++++++++++++- docs/adr/001-privacy-core-contract.md | 87 +- docs/privacy-capability-matrix.md | 4 +- docs/privacy-operations-roadmap.md | 54 +- scripts/test-node-package.mjs | 49 + scripts/test-wasm-package.mjs | 35 +- 16 files changed, 2593 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e8870c3..c016a1a 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,15 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based detectors currently report no confidence score. -The transformation strategies are `redact`, `mask`, `remove`, and -`pseudonymize` in Rust, Python, and Node.js. +The transformation strategies are `redact`, `mask`, `remove`, `pseudonymize`, +and `tokenize` in Rust, Python, and Node.js. Redaction uses an unnumbered `[ENTITY_TYPE]` placeholder, masking supports full or leading/trailing reveal modes, and removal deletes only the exact finding span. Pseudonymization uses provider-resolved 256-bit keys and deterministic -HMAC-SHA-256 tokens; it is deliberately unsupported in browser WASM. +HMAC-SHA-256 tokens. Tokenization uses an application-supplied asynchronous +provider to issue opaque `DFTOKENv1(...)` envelopes and restore them under an +exact request-level scope. Both provider-backed strategies are deliberately +unsupported in browser WASM. `transform` requires explicit findings; `scan_and_transform` (or `scanAndTransform` in JavaScript) is the explicit scan-then-transform convenience. Results include the transformed text and an ordered record for @@ -134,6 +137,17 @@ async def pseudonymize(): ) pseudonymized = asyncio.run(pseudonymize()) + +# A token provider implements tokenize_batch(scope, items) and +# restore_batch(scope, items). It owns storage or reversible cryptography, +# authorization, lifecycle, and audit. +token_manager = PrivacyManager(None, token_provider=TokenProvider()) +tokenized = asyncio.run(token_manager.scan_and_transform( + "Email jane@example.com", + {"transform": {"default": {"strategy": "tokenize", "token_ref": "customers/default"}}}, + {"scope": "tenant-a"}, +)) +restored = asyncio.run(token_manager.restore(tokenized.text, {"scope": "tenant-a"})) ``` ### Node.js @@ -160,6 +174,14 @@ const pseudonymized = await manager.scanAndTransform("Email jane@example.com", { default: { strategy: "pseudonymize", key_ref: "customers/email" }, }, }); + +const tokenManager = new PrivacyManager({ tokenProvider }); +const tokenized = await tokenManager.scanAndTransform( + "Email jane@example.com", + { transform: { default: { strategy: "tokenize", token_ref: "customers/default" } } }, + { scope: "tenant-a" }, +); +const restored = await tokenManager.restore(tokenized.text, { scope: "tenant-a" }); ``` The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64. diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index b73e5bf..14cebfb 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -5,7 +5,8 @@ export type TransformationStrategy = | "redact" | "mask" | "remove" - | "pseudonymize"; + | "pseudonymize" + | "tokenize"; export interface MaskRevealConfig { readonly direction: "first" | "last"; @@ -20,6 +21,10 @@ export type TransformationStrategyConfig = readonly key_ref: string; readonly key_version?: string; } + | { + readonly strategy: "tokenize"; + readonly token_ref: string; + } | { readonly strategy: "mask"; readonly character?: string; @@ -61,6 +66,15 @@ export type DataFogErrorCode = | "key_provider_unavailable" | "invalid_key_material" | "key_provider_error" + | "token_provider_required" + | "invalid_token" + | "unsupported_token_version" + | "token_not_found" + | "token_expired" + | "token_access_denied" + | "invalid_token_material" + | "token_provider_unavailable" + | "token_provider_error" | "unsupported_strategy" | "internal_error"; @@ -85,15 +99,56 @@ export interface KeyProvider { resolveKey(request: KeyProviderRequest): Promise; } +export interface PrivacyContext { + readonly scope: string; +} + +export interface TokenizeProviderItem { + readonly id: string; + readonly exactValue: string; + readonly tokenRef: string; +} + +export interface TokenizeProviderResult { + readonly id: string; + readonly payload: Uint8Array; + readonly resolvedVersion: string; +} + +export interface RestoreProviderItem { + readonly id: string; + readonly tokenRef: string; + readonly resolvedVersion: string; + readonly payload: Uint8Array; +} + +export interface RestoreProviderResult { + readonly id: string; + readonly value: string; +} + +export interface TokenProvider { + tokenizeBatch(scope: string, items: TokenizeProviderItem[]): Promise; + restoreBatch(scope: string, items: RestoreProviderItem[]): Promise; +} + +export interface PrivacyManagerProviders { + readonly keyProvider?: KeyProvider; + readonly tokenProvider?: TokenProvider; +} + export declare class PrivacyManager { - constructor(provider: KeyProvider); + constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, findings: Finding[], config: TransformationConfig, + context?: PrivacyContext, ): Promise; scanAndTransform( text: string, config: ScanAndTransformConfig, + context?: PrivacyContext, ): Promise; + restore(text: string, context: PrivacyContext): Promise; } diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 3fd38a7..75b5452 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -5,7 +5,8 @@ export type TransformationStrategy = | "redact" | "mask" | "remove" - | "pseudonymize"; + | "pseudonymize" + | "tokenize"; export interface MaskRevealConfig { readonly direction: "first" | "last"; @@ -20,6 +21,10 @@ export type TransformationStrategyConfig = readonly key_ref: string; readonly key_version?: string; } + | { + readonly strategy: "tokenize"; + readonly token_ref: string; + } | { readonly strategy: "mask"; readonly character?: string; @@ -61,6 +66,15 @@ export type DataFogErrorCode = | "key_provider_unavailable" | "invalid_key_material" | "key_provider_error" + | "token_provider_required" + | "invalid_token" + | "unsupported_token_version" + | "token_not_found" + | "token_expired" + | "token_access_denied" + | "invalid_token_material" + | "token_provider_unavailable" + | "token_provider_error" | "unsupported_strategy" | "internal_error"; @@ -85,17 +99,58 @@ export interface KeyProvider { resolveKey(request: KeyProviderRequest): Promise; } +export interface PrivacyContext { + readonly scope: string; +} + +export interface TokenizeProviderItem { + readonly id: string; + readonly exactValue: string; + readonly tokenRef: string; +} + +export interface TokenizeProviderResult { + readonly id: string; + readonly payload: Uint8Array; + readonly resolvedVersion: string; +} + +export interface RestoreProviderItem { + readonly id: string; + readonly tokenRef: string; + readonly resolvedVersion: string; + readonly payload: Uint8Array; +} + +export interface RestoreProviderResult { + readonly id: string; + readonly value: string; +} + +export interface TokenProvider { + tokenizeBatch(scope: string, items: TokenizeProviderItem[]): Promise; + restoreBatch(scope: string, items: RestoreProviderItem[]): Promise; +} + +export interface PrivacyManagerProviders { + readonly keyProvider?: KeyProvider; + readonly tokenProvider?: TokenProvider; +} + export declare class PrivacyManager { - constructor(provider: KeyProvider); + constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, findings: Finding[], config: TransformationConfig, + context?: PrivacyContext, ): Promise; scanAndTransform( text: string, config: ScanAndTransformConfig, + context?: PrivacyContext, ): Promise; + restore(text: string, context: PrivacyContext): Promise; } export interface Finding { readonly entityType: EntityType @@ -123,12 +178,44 @@ export declare function prepareScanAndTransform(text: string, config: ScanAndTra export declare function requiredKeySelectors(text: string, findings: Array, config: TransformationConfig): Array +export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array + +export declare function requiredTokenizationItems(text: string, findings: Array, config: TransformationConfig, context?: PrivacyContext | undefined): Array + export interface ResolvedKeyInput { selectorIndex: number key: Uint8Array resolvedVersion: string } +export interface Restoration { + readonly sourceByteRange: TextRange + readonly sourceCodepointRange: TextRange + readonly outputByteRange: TextRange + readonly outputCodepointRange: TextRange + readonly tokenRef: string + readonly resolvedTokenVersion: string +} + +export interface RestoredValueInput { + id: string + value: string +} + +export interface RestoreItem { + readonly id: string + readonly tokenRef: string + readonly resolvedVersion: string + readonly payload: Uint8Array +} + +export interface RestoreResult { + readonly text: string + readonly restorations: Array +} + +export declare function restoreWithResults(text: string, context: PrivacyContext, results: Array): RestoreResult + /** Scan text for supported PII findings. */ export declare function scan(text: string, config?: ScanConfig | undefined): Array @@ -140,6 +227,18 @@ export interface TextRange { readonly end: number } +export interface TokenizeItem { + readonly id: string + readonly exactValue: string + readonly tokenRef: string +} + +export interface TokenizeResultInput { + id: string + payload: Uint8Array + resolvedVersion: string +} + /** Transform explicit findings without scanning implicitly. */ export declare function transform(text: string, findings: Array, config: TransformationConfig): TransformResult @@ -156,6 +255,8 @@ export interface Transformation { readonly outputCodepointRange: TextRange readonly keyRef?: string readonly resolvedKeyVersion?: string + readonly tokenRef?: string + readonly resolvedTokenVersion?: string } export interface TransformResult { @@ -163,4 +264,6 @@ export interface TransformResult { readonly transformations: Array } +export declare function transformWithProviderResults(text: string, findings: Array, config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array, tokenResults: Array): TransformResult + export declare function transformWithResolvedKeys(text: string, findings: Array, config: TransformationConfig, resolvedKeys: Array): TransformResult diff --git a/bindings/node/index.js b/bindings/node/index.js index 0f52c25..1279f21 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -2,10 +2,13 @@ import { Buffer } from "node:buffer"; import { prepareScanAndTransform as nativePrepareScanAndTransform, requiredKeySelectors as nativeRequiredKeySelectors, + requiredRestoreItems as nativeRequiredRestoreItems, + requiredTokenizationItems as nativeRequiredTokenizationItems, + restoreWithResults as nativeRestoreWithResults, scan as nativeScan, scanAndTransform as nativeScanAndTransform, transform as nativeTransform, - transformWithResolvedKeys as nativeTransformWithResolvedKeys, + transformWithProviderResults as nativeTransformWithProviderResults, } from "./native.js"; export class DataFogError extends Error { @@ -45,14 +48,22 @@ const providerErrorMessages = { key_access_denied: "key provider denied access to the requested key", key_provider_unavailable: "key provider is temporarily unavailable", key_provider_error: "key provider could not resolve the requested key", + token_not_found: "token was not found", + token_expired: "token has expired", + token_access_denied: "token access was denied", + token_provider_unavailable: "token provider is temporarily unavailable", + token_provider_error: "token provider could not complete the request", }; -function normalizeProviderError(error, path) { +function normalizeProviderError(error, path, fallback = "key_provider_error") { const candidateCode = error?.code; + const expectedPrefix = fallback.startsWith("token_") ? "token_" : "key_"; const code = - typeof candidateCode === "string" && candidateCode in providerErrorMessages + typeof candidateCode === "string" && + candidateCode.startsWith(expectedPrefix) && + candidateCode in providerErrorMessages ? candidateCode - : "key_provider_error"; + : fallback; return new DataFogError({ code, message: providerErrorMessages[code], @@ -71,12 +82,6 @@ function withPathPrefix(error, prefix) { }); } -function assertProvider(provider) { - if (!provider || typeof provider.resolveKey !== "function") { - throw new TypeError("PrivacyManager provider must define resolveKey(request)"); - } -} - function resolvedKeyInput(selector, response) { if ( !response || @@ -97,19 +102,38 @@ function resolvedKeyInput(selector, response) { } export class PrivacyManager { - #provider; + #keyProvider; + #tokenProvider; - constructor(provider) { - assertProvider(provider); - this.#provider = provider; + constructor(provider, tokenProvider) { + const options = provider && ("keyProvider" in provider || "tokenProvider" in provider) + ? provider + : { keyProvider: provider, tokenProvider }; + if (options.keyProvider && typeof options.keyProvider.resolveKey !== "function") { + throw new TypeError("key provider must define resolveKey(request)"); + } + if (options.tokenProvider && + (typeof options.tokenProvider.tokenizeBatch !== "function" || + typeof options.tokenProvider.restoreBatch !== "function")) { + throw new TypeError("token provider must define tokenizeBatch(scope, items) and restoreBatch(scope, items)"); + } + this.#keyProvider = options.keyProvider; + this.#tokenProvider = options.tokenProvider; } async #resolve(selectors, pathPrefix = "") { const resolved = []; + if (selectors.length > 0 && !this.#keyProvider) { + throw new DataFogError({ + code: "key_provider_required", + message: "pseudonymization requires a runtime key provider", + path: `${pathPrefix}${selectors[0].path}`, + }); + } for (const selector of selectors) { let response; try { - response = await this.#provider.resolveKey({ + response = await this.#keyProvider.resolveKey({ keyRef: selector.keyRef, keyVersion: selector.keyVersion, }); @@ -121,7 +145,29 @@ export class PrivacyManager { return resolved; } - async transform(text, findings, config) { + async #tokenize(items, context) { + if (items.length === 0) return []; + if (!this.#tokenProvider) { + throw new DataFogError({ + code: "token_provider_required", + message: "tokenization requires a runtime token provider and request scope", + }); + } + let results; + try { + results = await this.#tokenProvider.tokenizeBatch(context.scope, items); + } catch (error) { + throw normalizeProviderError(error, undefined, "token_provider_error"); + } + if (!Array.isArray(results)) return []; + return results.map((result) => ({ + id: result?.id ?? "", + payload: result?.payload instanceof Uint8Array ? Buffer.from(result.payload) : Buffer.alloc(0), + resolvedVersion: typeof result?.resolvedVersion === "string" ? result.resolvedVersion : "", + })); + } + + async transform(text, findings, config, context) { if (typeof text !== "string") { throw new TypeError("transform text must be a string"); } @@ -132,7 +178,9 @@ export class PrivacyManager { try { const selectors = nativeRequiredKeySelectors(text, findings, config); resolved = await this.#resolve(selectors); - return nativeTransformWithResolvedKeys(text, findings, config, resolved); + const items = nativeRequiredTokenizationItems(text, findings, config, context); + const tokens = await this.#tokenize(items, context); + return nativeTransformWithProviderResults(text, findings, config, context, resolved, tokens); } catch (error) { if (error instanceof DataFogError) throw error; throw normalizeError(error, "invalid_configuration"); @@ -141,7 +189,7 @@ export class PrivacyManager { } } - async scanAndTransform(text, config) { + async scanAndTransform(text, config, context) { if (typeof text !== "string") { throw new TypeError("scanAndTransform text must be a string"); } @@ -153,11 +201,15 @@ export class PrivacyManager { } const resolved = await this.#resolve(prepared.selectors, "/transform"); try { - return nativeTransformWithResolvedKeys( + const items = nativeRequiredTokenizationItems(text, prepared.findings, config.transform, context); + const tokens = await this.#tokenize(items, context); + return nativeTransformWithProviderResults( text, prepared.findings, config.transform, + context, resolved, + tokens, ); } catch (error) { throw withPathPrefix(error, "/transform"); @@ -165,6 +217,33 @@ export class PrivacyManager { resolved.forEach(({ key }) => key.fill(0)); } } + + + async restore(text, context) { + if (typeof text !== "string") { + throw new TypeError("restore text must be a string"); + } + let items; + try { + items = nativeRequiredRestoreItems(text, context); + } catch (error) { + throw normalizeError(error, "invalid_configuration"); + } + if (items.length === 0) return nativeRestoreWithResults(text, context, []); + if (!this.#tokenProvider) { + throw new DataFogError({ + code: "token_provider_required", + message: "restoration requires a runtime token provider", + }); + } + let results; + try { + results = await this.#tokenProvider.restoreBatch(context.scope, items); + } catch (error) { + throw normalizeProviderError(error, undefined, "token_provider_error"); + } + return nativeRestoreWithResults(text, context, Array.isArray(results) ? results : []); + } } export function scan(text, config) { diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 90e2ad9..2ebbe27 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -74,6 +74,12 @@ pub struct Transformation { #[napi(readonly)] pub resolved_key_version: Option, + + #[napi(readonly)] + pub token_ref: Option, + + #[napi(readonly)] + pub resolved_token_version: Option, } #[napi(object, object_from_js = false)] @@ -108,6 +114,66 @@ pub struct ResolvedKeyInput { pub resolved_version: String, } +#[napi(object, object_from_js = false)] +pub struct TokenizeItem { + #[napi(readonly)] + pub id: String, + #[napi(readonly)] + pub exact_value: String, + #[napi(readonly)] + pub token_ref: String, +} + +#[napi(object)] +pub struct TokenizeResultInput { + pub id: String, + #[napi(ts_type = "Uint8Array")] + pub payload: Buffer, + pub resolved_version: String, +} + +#[napi(object, object_from_js = false)] +pub struct RestoreItem { + #[napi(readonly)] + pub id: String, + #[napi(readonly)] + pub token_ref: String, + #[napi(readonly)] + pub resolved_version: String, + #[napi(readonly, ts_type = "Uint8Array")] + pub payload: Buffer, +} + +#[napi(object)] +pub struct RestoredValueInput { + pub id: String, + pub value: String, +} + +#[napi(object, object_from_js = false)] +pub struct Restoration { + #[napi(readonly)] + pub source_byte_range: TextRange, + #[napi(readonly)] + pub source_codepoint_range: TextRange, + #[napi(readonly)] + pub output_byte_range: TextRange, + #[napi(readonly)] + pub output_codepoint_range: TextRange, + #[napi(readonly)] + pub token_ref: String, + #[napi(readonly)] + pub resolved_token_version: String, +} + +#[napi(object, object_from_js = false)] +pub struct RestoreResult { + #[napi(readonly)] + pub text: String, + #[napi(readonly)] + pub restorations: Vec, +} + #[napi(object, object_from_js = false)] pub struct PreparedScanAndTransform { #[napi(readonly)] @@ -201,12 +267,48 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result { "pseudonymize".to_owned() } + datafog_core::TransformationStrategy::Tokenize(_) => "tokenize".to_owned(), }, replacement: transformation.replacement, output_byte_range: js_range(transformation.output_byte_range)?, output_codepoint_range: js_range(transformation.output_codepoint_range)?, key_ref: transformation.key_ref, resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, + }) + }) + .collect::>>()?, + }) +} + +fn core_token_results(results: Vec) -> Vec { + results + .into_iter() + .map(|result| { + datafog_core::TokenizeResult::new( + result.id, + result.payload.to_vec(), + result.resolved_version, + ) + }) + .collect() +} + +fn js_restore_result(result: datafog_core::RestoreResult) -> napi::Result { + Ok(RestoreResult { + text: result.text, + restorations: result + .restorations + .into_iter() + .map(|record| { + Ok(Restoration { + source_byte_range: js_range(record.source_byte_range)?, + source_codepoint_range: js_range(record.source_codepoint_range)?, + output_byte_range: js_range(record.output_byte_range)?, + output_codepoint_range: js_range(record.output_codepoint_range)?, + token_ref: record.token_ref, + resolved_token_version: record.resolved_token_version, }) }) .collect::>>()?, @@ -354,3 +456,111 @@ pub fn prepare_scan_and_transform( selectors: js_key_selectors(&selectors)?, }) } + +#[napi(strict, catch_unwind)] +pub fn required_tokenization_items( + env: Env, + text: String, + findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, + #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, +) -> napi::Result> { + let config_value: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_transformation_config(&config_value).map_err(js_privacy_error)?; + let context = context + .map(|value| -> napi::Result { env.from_js_value(value) }) + .transpose()? + .map(|value| datafog_core::parse_privacy_context(&value)) + .transpose() + .map_err(js_privacy_error)?; + let findings = findings.into_iter().map(core_finding).collect::>(); + datafog_core::required_tokenization_items(&text, &findings, &config, context.as_ref()) + .map_err(js_privacy_error) + .map(|items| { + items + .into_iter() + .map(|item| TokenizeItem { + id: item.id().to_owned(), + exact_value: item.exact_value().to_owned(), + token_ref: item.token_ref().to_owned(), + }) + .collect() + }) +} + +#[napi(strict, catch_unwind)] +pub fn transform_with_provider_results( + env: Env, + text: String, + findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, + #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, + resolved_keys: Vec, + token_results: Vec, +) -> napi::Result { + let config_value: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_transformation_config(&config_value).map_err(js_privacy_error)?; + let context = context + .map(|value| -> napi::Result { env.from_js_value(value) }) + .transpose()? + .map(|value| datafog_core::parse_privacy_context(&value)) + .transpose() + .map_err(js_privacy_error)?; + let findings = findings.into_iter().map(core_finding).collect::>(); + let selectors = datafog_core::required_key_selectors(&text, &findings, &config) + .map_err(js_privacy_error)?; + let keys = core_key_bindings(selectors, resolved_keys)?; + datafog_core::transform_with_provider_results( + &text, + &findings, + &config, + context.as_ref(), + keys, + core_token_results(token_results), + ) + .map_err(js_privacy_error) + .and_then(js_transform_result) +} + +#[napi(strict, catch_unwind)] +pub fn required_restore_items( + env: Env, + text: String, + #[napi(ts_arg_type = "PrivacyContext")] context: Unknown<'_>, +) -> napi::Result> { + let value: serde_json::Value = env.from_js_value(context)?; + let context = datafog_core::parse_privacy_context(&value).map_err(js_privacy_error)?; + datafog_core::required_restore_items(&text, &context) + .map_err(js_privacy_error) + .map(|items| { + items + .into_iter() + .map(|item| RestoreItem { + id: item.id().to_owned(), + token_ref: item.token_ref().to_owned(), + resolved_version: item.resolved_version().to_owned(), + payload: Buffer::from(item.payload()), + }) + .collect() + }) +} + +#[napi(strict, catch_unwind)] +pub fn restore_with_results( + env: Env, + text: String, + #[napi(ts_arg_type = "PrivacyContext")] context: Unknown<'_>, + results: Vec, +) -> napi::Result { + let value: serde_json::Value = env.from_js_value(context)?; + let context = datafog_core::parse_privacy_context(&value).map_err(js_privacy_error)?; + let results = results + .into_iter() + .map(|result| datafog_core::RestoredValue::new(result.id, result.value)) + .collect(); + datafog_core::restore_with_results(&text, &context, results) + .map_err(js_privacy_error) + .and_then(js_restore_result) +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index c9d3855..bb3774e 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -202,6 +202,12 @@ struct Transformation { #[pyo3(get)] resolved_key_version: Option, + + #[pyo3(get)] + token_ref: Option, + + #[pyo3(get)] + resolved_token_version: Option, } impl From for Transformation { @@ -218,12 +224,15 @@ impl From for Transformation { core::TransformationStrategy::Remove => "remove".to_owned(), core::TransformationStrategy::Mask(_) => "mask".to_owned(), core::TransformationStrategy::Pseudonymize(_) => "pseudonymize".to_owned(), + core::TransformationStrategy::Tokenize(_) => "tokenize".to_owned(), }, replacement: transformation.replacement, output_byte_range: transformation.output_byte_range.into(), output_codepoint_range: transformation.output_codepoint_range.into(), key_ref: transformation.key_ref, resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, } } } @@ -243,6 +252,8 @@ impl Transformation { && self.output_codepoint_range == other.output_codepoint_range && self.key_ref == other.key_ref && self.resolved_key_version == other.resolved_key_version + && self.token_ref == other.token_ref + && self.resolved_token_version == other.resolved_token_version } } @@ -277,8 +288,53 @@ impl TransformResult { } } +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct Restoration { + #[pyo3(get)] + source_byte_range: TextRange, + #[pyo3(get)] + source_codepoint_range: TextRange, + #[pyo3(get)] + output_byte_range: TextRange, + #[pyo3(get)] + output_codepoint_range: TextRange, + #[pyo3(get)] + token_ref: String, + #[pyo3(get)] + resolved_token_version: String, +} + +#[pyclass(frozen, skip_from_py_object)] +struct RestoreResult { + #[pyo3(get)] + text: String, + #[pyo3(get)] + restorations: Vec, +} + +impl From for RestoreResult { + fn from(result: core::RestoreResult) -> Self { + Self { + text: result.text, + restorations: result + .restorations + .into_iter() + .map(|record| Restoration { + source_byte_range: record.source_byte_range.into(), + source_codepoint_range: record.source_codepoint_range.into(), + output_byte_range: record.output_byte_range.into(), + output_codepoint_range: record.output_codepoint_range.into(), + token_ref: record.token_ref, + resolved_token_version: record.resolved_token_version, + }) + .collect(), + } + } +} + struct PythonKeyProvider { - provider: Py, + provider: Option>, } fn provider_error_kind(error: &PyErr) -> core::KeyProviderErrorKind { @@ -308,6 +364,18 @@ fn provider_field<'py>( } } +fn required_provider_string(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + provider_field(value, name)? + .ok_or_else(|| PyErr::new::(format!("provider response requires {name}")))? + .extract() +} + +fn required_provider_bytes(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + provider_field(value, name)? + .ok_or_else(|| PyErr::new::(format!("provider response requires {name}")))? + .extract() +} + fn resolved_key_from_python(value: &Bound<'_, PyAny>) -> core::ResolvedKey { let key = provider_field(value, "key") .ok() @@ -323,11 +391,21 @@ fn resolved_key_from_python(value: &Bound<'_, PyAny>) -> core::ResolvedKey { } impl core::KeyProvider for PythonKeyProvider { + fn is_configured(&self) -> bool { + self.provider.is_some() + } + fn resolve_key(&self, selector: core::KeySelector) -> core::KeyProviderFuture<'_> { - let provider = Python::attach(|py| self.provider.clone_ref(py)); + let provider = Python::attach(|py| { + self.provider + .as_ref() + .map(|provider| provider.clone_ref(py)) + }); Box::pin(async move { let future = Python::attach(|py| { let awaitable = provider + .as_ref() + .ok_or_else(|| PyErr::new::("key provider is required"))? .bind(py) .call_method1("resolve_key", (selector.key_ref(), selector.key_version()))?; pyo3_async_runtimes::tokio::into_future(awaitable) @@ -343,33 +421,202 @@ impl core::KeyProvider for PythonKeyProvider { } } +struct PythonTokenProvider { + provider: Option>, +} + +fn token_provider_error_kind(error: &PyErr) -> core::TokenProviderErrorKind { + Python::attach(|py| { + let code = error + .value(py) + .getattr("code") + .and_then(|value| value.extract::()) + .ok(); + match code.as_deref() { + Some("token_not_found") => core::TokenProviderErrorKind::NotFound, + Some("token_expired") => core::TokenProviderErrorKind::Expired, + Some("token_access_denied") => core::TokenProviderErrorKind::AccessDenied, + Some("token_provider_unavailable") => core::TokenProviderErrorKind::Unavailable, + _ => core::TokenProviderErrorKind::ProviderError, + } + }) +} + +impl core::TokenProvider for PythonTokenProvider { + fn is_configured(&self) -> bool { + self.provider.is_some() + } + + fn tokenize_batch( + &self, + scope: &str, + items: Vec, + ) -> core::TokenizeProviderFuture<'_> { + let provider = Python::attach(|py| { + self.provider + .as_ref() + .map(|provider| provider.clone_ref(py)) + }); + let scope = scope.to_owned(); + Box::pin(async move { + let future = Python::attach(|py| -> PyResult<_> { + let provider = provider + .as_ref() + .ok_or_else(|| PyErr::new::("token provider is required"))?; + let requests = PyList::empty(py); + for item in items { + let request = PyDict::new(py); + request.set_item("id", item.id())?; + request.set_item("exact_value", item.exact_value())?; + request.set_item("token_ref", item.token_ref())?; + requests.append(request)?; + } + let awaitable = provider + .bind(py) + .call_method1("tokenize_batch", (scope, requests))?; + pyo3_async_runtimes::tokio::into_future(awaitable) + }) + .map_err(|error| core::TokenProviderError::new(token_provider_error_kind(&error)))?; + let response = future.await.map_err(|error| { + core::TokenProviderError::new(token_provider_error_kind(&error)) + })?; + Ok(Python::attach(|py| { + let Ok(values) = response.bind(py).try_iter() else { + return vec![core::TokenizeResult::new("", Vec::new(), "")]; + }; + let mut parsed = Vec::new(); + for value in values { + let Ok(value) = value else { + return vec![core::TokenizeResult::new("", Vec::new(), "")]; + }; + let parsed_fields = ( + required_provider_string(&value, "id"), + required_provider_bytes(&value, "payload"), + required_provider_string(&value, "resolved_version"), + ); + let (Ok(id), Ok(payload), Ok(version)) = parsed_fields else { + return vec![core::TokenizeResult::new("", Vec::new(), "")]; + }; + parsed.push(core::TokenizeResult::new(id, payload, version)); + } + parsed + })) + }) + } + + fn restore_batch( + &self, + scope: &str, + items: Vec, + ) -> core::RestoreProviderFuture<'_> { + let provider = Python::attach(|py| { + self.provider + .as_ref() + .map(|provider| provider.clone_ref(py)) + }); + let scope = scope.to_owned(); + Box::pin(async move { + let future = Python::attach(|py| -> PyResult<_> { + let provider = provider + .as_ref() + .ok_or_else(|| PyErr::new::("token provider is required"))?; + let requests = PyList::empty(py); + for item in items { + let request = PyDict::new(py); + request.set_item("id", item.id())?; + request.set_item("token_ref", item.token_ref())?; + request.set_item("resolved_version", item.resolved_version())?; + request.set_item("payload", item.payload())?; + requests.append(request)?; + } + let awaitable = provider + .bind(py) + .call_method1("restore_batch", (scope, requests))?; + pyo3_async_runtimes::tokio::into_future(awaitable) + }) + .map_err(|error| core::TokenProviderError::new(token_provider_error_kind(&error)))?; + let response = future.await.map_err(|error| { + core::TokenProviderError::new(token_provider_error_kind(&error)) + })?; + Ok(Python::attach(|py| { + let Ok(values) = response.bind(py).try_iter() else { + return vec![core::RestoredValue::new("", "")]; + }; + let mut parsed = Vec::new(); + for value in values { + let Ok(value) = value else { + return vec![core::RestoredValue::new("", "")]; + }; + let parsed_fields = ( + required_provider_string(&value, "id"), + required_provider_string(&value, "value"), + ); + let (Ok(id), Ok(restored)) = parsed_fields else { + return vec![core::RestoredValue::new("", "")]; + }; + parsed.push(core::RestoredValue::new(id, restored)); + } + parsed + })) + }) + } +} + /// Provider-backed asynchronous privacy manager. #[pyclass(frozen, skip_from_py_object)] struct PrivacyManager { - provider: Py, + key_provider: Option>, + token_provider: Option>, } #[pymethods] impl PrivacyManager { #[new] - fn new(py: Python<'_>, provider: Py) -> PyResult { - let resolve_key = provider.bind(py).getattr("resolve_key").map_err(|_| { - PyErr::new::("provider must define resolve_key(key_ref, key_version)") - })?; - if !resolve_key.is_callable() { - return Err(PyErr::new::( - "provider resolve_key attribute must be callable", - )); + #[pyo3(signature = (provider=None, token_provider=None))] + fn new( + py: Python<'_>, + provider: Option>, + token_provider: Option>, + ) -> PyResult { + if let Some(provider) = &provider { + let resolve_key = provider.bind(py).getattr("resolve_key").map_err(|_| { + PyErr::new::( + "key provider must define resolve_key(key_ref, key_version)", + ) + })?; + if !resolve_key.is_callable() { + return Err(PyErr::new::( + "provider resolve_key attribute must be callable", + )); + } } - Ok(Self { provider }) + if let Some(provider) = &token_provider { + for method in ["tokenize_batch", "restore_batch"] { + if !provider + .bind(py) + .getattr(method) + .is_ok_and(|value| value.is_callable()) + { + return Err(PyErr::new::( + "token provider must define tokenize_batch and restore_batch", + )); + } + } + } + Ok(Self { + key_provider: provider, + token_provider, + }) } + #[pyo3(signature = (text, findings, config, context=None))] fn transform<'py>( &self, py: Python<'py>, text: String, findings: Vec>, config: Py, + context: Option>, ) -> PyResult> { let config_value = py_to_json(py, config.bind(py), "")?; let config = core::parse_transformation_config(&config_value) @@ -378,36 +625,106 @@ impl PrivacyManager { .iter() .map(|finding| finding.bind(py).borrow().to_core()) .collect::>(); - let provider = self.provider.clone_ref(py); + let context = context + .map(|context| py_to_json(py, context.bind(py), "")) + .transpose()? + .map(|value| core::parse_privacy_context(&value)) + .transpose() + .map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let manager = core::PrivacyManager::new(PythonKeyProvider { provider }); + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); let result = manager - .transform(&text, &findings, &config) + .transform_with_context(&text, &findings, &config, context.as_ref()) .await .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; Python::attach(|py| Py::new(py, TransformResult::from(result))) }) } + #[pyo3(signature = (text, config, context=None))] fn scan_and_transform<'py>( &self, py: Python<'py>, text: String, config: Py, + context: Option>, ) -> PyResult> { let config_value = py_to_json(py, config.bind(py), "")?; let config = core::parse_scan_and_transform_config(&config_value) .map_err(|error| privacy_error(py, error))?; - let provider = self.provider.clone_ref(py); + let context = context + .map(|context| py_to_json(py, context.bind(py), "")) + .transpose()? + .map(|value| core::parse_privacy_context(&value)) + .transpose() + .map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); pyo3_async_runtimes::tokio::future_into_py(py, async move { - let manager = core::PrivacyManager::new(PythonKeyProvider { provider }); + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); let result = manager - .scan_and_transform(&text, &config) + .scan_and_transform_with_context(&text, &config, context.as_ref()) .await .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; Python::attach(|py| Py::new(py, TransformResult::from(result))) }) } + + fn restore<'py>( + &self, + py: Python<'py>, + text: String, + context: Py, + ) -> PyResult> { + let context = py_to_json(py, context.bind(py), "")?; + let context = + core::parse_privacy_context(&context).map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); + let result = manager + .restore(&text, &context) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, RestoreResult::from(result))) + }) + } } fn configuration_conversion_error(py: Python<'_>, path: &str, message: &str) -> PyErr { @@ -501,6 +818,15 @@ fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr { | core::PrivacyErrorCode::KeyProviderUnavailable | core::PrivacyErrorCode::InvalidKeyMaterial | core::PrivacyErrorCode::KeyProviderError + | core::PrivacyErrorCode::TokenProviderRequired + | core::PrivacyErrorCode::InvalidToken + | core::PrivacyErrorCode::UnsupportedTokenVersion + | core::PrivacyErrorCode::TokenNotFound + | core::PrivacyErrorCode::TokenExpired + | core::PrivacyErrorCode::TokenAccessDenied + | core::PrivacyErrorCode::InvalidTokenMaterial + | core::PrivacyErrorCode::TokenProviderUnavailable + | core::PrivacyErrorCode::TokenProviderError | core::PrivacyErrorCode::UnsupportedStrategy => { PyErr::new::(error.to_string()) } @@ -595,6 +921,8 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_class::()?; module.add_function(wrap_pyfunction!(scan, module)?)?; module.add_function(wrap_pyfunction!(transform, module)?)?; diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index 74a0835..379be76 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -267,6 +267,76 @@ async def provider_transform(active_provider: Provider): else: raise AssertionError("invalid provider key material was accepted") + class TokenProvider: + def __init__(self) -> None: + self.next_payload = 0 + self.records: dict[bytes, tuple[str, str, str, str]] = {} + + async def tokenize_batch(self, scope: str, items: list[dict[str, object]]): + results = [] + for item in items: + self.next_payload += 1 + payload = bytes([self.next_payload]) + self.records[payload] = ( + scope, + str(item["token_ref"]), + "active-1", + str(item["exact_value"]), + ) + results.append( + { + "id": item["id"], + "payload": payload, + "resolved_version": "active-1", + } + ) + return results + + async def restore_batch(self, scope: str, items: list[dict[str, object]]): + results = [] + for item in items: + record = self.records.get(bytes(item["payload"])) + if record is None or record[:3] != ( + scope, + item["token_ref"], + item["resolved_version"], + ): + error = RuntimeError("denied") + error.code = "token_access_denied" + raise error + results.append({"id": item["id"], "value": record[3]}) + return results + + async def token_round_trip(): + manager = PrivacyManager(None, TokenProvider()) + context = {"scope": "tenant/α"} + tokenized = await manager.scan_and_transform( + "👋 jane@example.com jane@example.com", + { + "transform": { + "default": { + "strategy": "tokenize", + "token_ref": "customers/default", + } + } + }, + context, + ) + assert tokenized.transformations[0].replacement != tokenized.transformations[1].replacement + assert tokenized.transformations[0].token_ref == "customers/default" + assert tokenized.transformations[0].resolved_token_version == "active-1" + restored = await manager.restore(tokenized.text, context) + assert restored.text == "👋 jane@example.com jane@example.com" + assert len(restored.restorations) == 2 + try: + await manager.restore(tokenized.text, {"scope": "tenant/b"}) + except DataFogKeyProviderError as error: + assert error.code == "token_access_denied" + else: + raise AssertionError("wrong-scope restoration was accepted") + + asyncio.run(token_round_trip()) + try: transform( text, diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index 4fc38a8..32b95f7 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -1,6 +1,6 @@ /** Canonical built-in values are uppercase, but custom detectors may add values. */ export type EntityType = string; -export type TransformationStrategy = "redact" | "mask" | "remove"; +export type TransformationStrategy = "redact" | "mask" | "remove" | "tokenize"; export interface MaskRevealConfig { readonly direction: "first" | "last"; @@ -10,6 +10,7 @@ export interface MaskRevealConfig { export type TransformationStrategyConfig = | { readonly strategy: "redact" } | { readonly strategy: "remove" } + | { readonly strategy: "tokenize"; readonly token_ref: string } | { readonly strategy: "mask"; readonly character?: string; @@ -51,6 +52,15 @@ export type DataFogErrorCode = | "key_provider_unavailable" | "invalid_key_material" | "key_provider_error" + | "token_provider_required" + | "invalid_token" + | "unsupported_token_version" + | "token_not_found" + | "token_expired" + | "token_access_denied" + | "invalid_token_material" + | "token_provider_unavailable" + | "token_provider_error" | "unsupported_strategy" | "internal_error"; @@ -89,6 +99,8 @@ export interface Transformation { readonly outputCodepointRange: TextRange; readonly keyRef?: string; readonly resolvedKeyVersion?: string; + readonly tokenRef?: string; + readonly resolvedTokenVersion?: string; } export interface TransformResult { @@ -107,3 +119,14 @@ export function scanAndTransform( text: string, config: ScanAndTransformConfig, ): TransformResult; +export interface PrivacyContext { readonly scope: string; } +export interface Restoration { + readonly sourceByteRange: TextRange; + readonly sourceCodepointRange: TextRange; + readonly outputByteRange: TextRange; + readonly outputCodepointRange: TextRange; + readonly tokenRef: string; + readonly resolvedTokenVersion: string; +} +export interface RestoreResult { readonly text: string; readonly restorations: Restoration[]; } +export function restore(text: string, context: PrivacyContext): RestoreResult; diff --git a/bindings/wasm/index.js b/bindings/wasm/index.js index 37df521..6b53ea1 100644 --- a/bindings/wasm/index.js +++ b/bindings/wasm/index.js @@ -1,6 +1,7 @@ import initWasm, { scan as scanWasm, scan_and_transform as scanAndTransformWasm, + restore as restoreWasm, transform as transformWasm, } from "./dist/datafog_wasm.js"; @@ -110,3 +111,15 @@ export function scanAndTransform(text, config) { throw normalizeError(error, "invalid_configuration"); } } + +export function restore(text, context) { + if (typeof text !== "string") { + throw new TypeError("restore text must be a string"); + } + assertInitialized("restore"); + try { + return restoreWasm(text, context); + } catch (error) { + throw normalizeError(error, "invalid_configuration"); + } +} diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index d088fcc..393a01c 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -63,6 +63,8 @@ struct Transformation { output_codepoint_range: TextRange, key_ref: Option, resolved_key_version: Option, + token_ref: Option, + resolved_token_version: Option, } #[derive(Serialize)] @@ -118,12 +120,15 @@ fn result_to_js(result: datafog_core::TransformResult) -> Result "remove", datafog_core::TransformationStrategy::Mask(_) => "mask", datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", + datafog_core::TransformationStrategy::Tokenize(_) => "tokenize", }, replacement: transformation.replacement, output_byte_range: transformation.output_byte_range.into(), output_codepoint_range: transformation.output_codepoint_range.into(), key_ref: transformation.key_ref, resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, }) .collect(), }; @@ -166,6 +171,20 @@ pub fn transform(text: &str, findings: JsValue, config: JsValue) -> Result Result Result { + let context = config_value(context)?; + let context = datafog_core::parse_privacy_context(&context).map_err(privacy_error)?; + datafog_core::required_restore_items(text, &context).map_err(privacy_error)?; + Err(privacy_error( + datafog_core::PrivacyError::unsupported_strategy("/restore"), + )) +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index ff3af29..a0165ee 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -50,6 +50,69 @@ pub enum TransformationStrategy { Mask(MaskConfig), /// Replace the exact finding value with a deterministic keyed pseudonym. Pseudonymize(PseudonymizeConfig), + /// Replace the exact finding value with an opaque reversible token. + Tokenize(TokenizeConfig), +} + +/// Provider-owned token profile selected by a tokenization request. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct TokenizeConfig { + token_ref: String, +} + +impl TokenizeConfig { + /// Create a validated token profile selector. + pub fn new(token_ref: impl Into) -> Result { + let token_ref = token_ref.into(); + if token_ref.trim().is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + "/token_ref", + "token_ref must not be empty or whitespace-only", + )); + } + Ok(Self { token_ref }) + } + + /// Provider-defined token profile reference. + pub fn token_ref(&self) -> &str { + &self.token_ref + } +} + +/// Non-secret request context used for authorization by token providers. +#[derive(Clone, PartialEq, Eq)] +pub struct PrivacyContext { + scope: String, +} + +impl std::fmt::Debug for PrivacyContext { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PrivacyContext") + .field("scope", &"[REDACTED]") + .finish() + } +} + +impl PrivacyContext { + /// Create exact, case-sensitive request context without normalization. + pub fn new(scope: impl Into) -> Result { + let scope = scope.into(); + if scope.trim().is_empty() { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + "/context/scope", + "scope must not be empty or whitespace-only", + )); + } + Ok(Self { scope }) + } + + /// Exact provider authorization scope. + pub fn scope(&self) -> &str { + &self.scope + } } /// Key selector for deterministic one-way pseudonymization. @@ -628,6 +691,21 @@ pub fn parse_scan_config(value: &serde_json::Value) -> Result Result { + let object = require_object(value, "/context", "context must be an object")?; + reject_unknown_fields(object, &["scope"], "/context")?; + let scope = object.get("scope").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + "/context/scope", + "token operations require scope", + ) + })?; + let scope = require_string(scope, "/context/scope", "scope must be a string")?; + PrivacyContext::new(scope) +} + fn parse_strategy_config( value: &serde_json::Value, path: &str, @@ -771,10 +849,32 @@ fn parse_strategy_config( } }) } + "tokenize" => { + reject_unknown_fields(object, &["strategy", "token_ref"], path)?; + let token_ref_path = format!("{path}/token_ref"); + let token_ref = object.get("token_ref").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + &token_ref_path, + "tokenization requires token_ref", + ) + })?; + let token_ref = + require_string(token_ref, &token_ref_path, "token_ref must be a string")?; + TokenizeConfig::new(token_ref) + .map(TransformationStrategy::Tokenize) + .map_err(|_| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::EmptyValue, + token_ref_path, + "token_ref must not be empty or whitespace-only", + ) + }) + } _ => Err(PrivacyError::invalid_configuration( PrivacyErrorReason::InvalidValue, strategy_path, - "strategy must be redact, mask, remove, or pseudonymize", + "strategy must be redact, mask, remove, pseudonymize, or tokenize", )), } } @@ -999,27 +1099,291 @@ pub type KeyProviderFuture<'a> = /// Runtime boundary for resolving pseudonymization key references. pub trait KeyProvider: Send + Sync { + /// Whether this value represents an available provider capability. + fn is_configured(&self) -> bool { + true + } + /// Resolve one key selector. Provider implementations own retries, /// timeouts, authentication, decoding, and optional caching. fn resolve_key(&self, selector: KeySelector) -> KeyProviderFuture<'_>; } +/// One source value submitted to a token provider. Equal values remain +/// separate items so providers may issue fresh tokens. +#[derive(Clone, PartialEq, Eq)] +pub struct TokenizeItem { + id: String, + exact_value: String, + token_ref: String, +} + +impl TokenizeItem { + /// Request-local correlation identifier. + pub fn id(&self) -> &str { + &self.id + } + + /// Exact selected source value. + pub fn exact_value(&self) -> &str { + &self.exact_value + } + + /// Provider-defined profile reference. + pub fn token_ref(&self) -> &str { + &self.token_ref + } +} + +impl std::fmt::Debug for TokenizeItem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TokenizeItem") + .field("id", &self.id) + .field("exact_value", &"[REDACTED]") + .field("token_ref", &self.token_ref) + .finish() + } +} + +/// Opaque token material returned for one tokenization item. +#[derive(Clone, PartialEq, Eq)] +pub struct TokenizeResult { + id: String, + payload: Vec, + resolved_version: String, +} + +impl TokenizeResult { + /// Construct an untrusted provider response for validation by the core. + pub fn new( + id: impl Into, + payload: Vec, + resolved_version: impl Into, + ) -> Self { + Self { + id: id.into(), + payload, + resolved_version: resolved_version.into(), + } + } + + /// Request-local correlation identifier. + pub fn id(&self) -> &str { + &self.id + } +} + +impl std::fmt::Debug for TokenizeResult { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TokenizeResult") + .field("id", &self.id) + .field("payload", &"[REDACTED]") + .field("resolved_version", &self.resolved_version) + .finish() + } +} + +/// One deduplicated canonical token submitted for restoration. +#[derive(Clone, PartialEq, Eq)] +pub struct RestoreItem { + id: String, + token_ref: String, + resolved_version: String, + payload: Vec, +} + +impl RestoreItem { + pub fn id(&self) -> &str { + &self.id + } + pub fn token_ref(&self) -> &str { + &self.token_ref + } + pub fn resolved_version(&self) -> &str { + &self.resolved_version + } + pub fn payload(&self) -> &[u8] { + &self.payload + } +} + +impl std::fmt::Debug for RestoreItem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RestoreItem") + .field("id", &self.id) + .field("token_ref", &self.token_ref) + .field("resolved_version", &self.resolved_version) + .field("payload", &"[REDACTED]") + .finish() + } +} + +/// Restored plaintext returned for one provider request item. +#[derive(Clone, PartialEq, Eq)] +pub struct RestoredValue { + id: String, + value: String, +} + +impl RestoredValue { + pub fn new(id: impl Into, value: impl Into) -> Self { + Self { + id: id.into(), + value: value.into(), + } + } + pub fn id(&self) -> &str { + &self.id + } +} + +impl std::fmt::Debug for RestoredValue { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RestoredValue") + .field("id", &self.id) + .field("value", &"[REDACTED]") + .finish() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TokenProviderErrorKind { + NotFound, + Expired, + AccessDenied, + Unavailable, + ProviderError, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TokenProviderError { + kind: TokenProviderErrorKind, +} + +impl TokenProviderError { + pub fn new(kind: TokenProviderErrorKind) -> Self { + Self { kind } + } + pub fn kind(self) -> TokenProviderErrorKind { + self.kind + } +} + +impl std::fmt::Display for TokenProviderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("token provider could not complete the request") + } +} + +impl std::error::Error for TokenProviderError {} + +pub type TokenizeProviderFuture<'a> = + Pin, TokenProviderError>> + Send + 'a>>; +pub type RestoreProviderFuture<'a> = + Pin, TokenProviderError>> + Send + 'a>>; + +/// Vendor-neutral asynchronous boundary for reversible token operations. +pub trait TokenProvider: Send + Sync { + /// Whether this value represents an available provider capability. + fn is_configured(&self) -> bool { + true + } + + fn tokenize_batch(&self, scope: &str, items: Vec) -> TokenizeProviderFuture<'_>; + fn restore_batch(&self, scope: &str, items: Vec) -> RestoreProviderFuture<'_>; +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct NoTokenProvider; + +#[derive(Debug, Clone, Copy, Default)] +pub struct NoKeyProvider; + +impl KeyProvider for NoKeyProvider { + fn is_configured(&self) -> bool { + false + } + + fn resolve_key(&self, _selector: KeySelector) -> KeyProviderFuture<'_> { + Box::pin(async { Err(KeyProviderError::new(KeyProviderErrorKind::ProviderError)) }) + } +} + +impl TokenProvider for NoTokenProvider { + fn is_configured(&self) -> bool { + false + } + + fn tokenize_batch( + &self, + _scope: &str, + _items: Vec, + ) -> TokenizeProviderFuture<'_> { + Box::pin(async { + Err(TokenProviderError::new( + TokenProviderErrorKind::ProviderError, + )) + }) + } + + fn restore_batch(&self, _scope: &str, _items: Vec) -> RestoreProviderFuture<'_> { + Box::pin(async { + Err(TokenProviderError::new( + TokenProviderErrorKind::ProviderError, + )) + }) + } +} + /// Provider-backed privacy operation manager. #[derive(Debug)] -pub struct PrivacyManager

{ +pub struct PrivacyManager { provider: P, + token_provider: T, } impl

PrivacyManager

{ /// Create a manager with one runtime provider. pub fn new(provider: P) -> Self { - Self { provider } + Self { + provider, + token_provider: NoTokenProvider, + } } /// Borrow the configured provider. pub fn provider(&self) -> &P { &self.provider } + + /// Add reversible token capability without changing the key provider. + pub fn with_token_provider(self, token_provider: T) -> PrivacyManager { + PrivacyManager { + provider: self.provider, + token_provider, + } + } +} + +impl PrivacyManager { + /// Create a manager that supports token operations but no pseudonymization. + pub fn token_provider_only(token_provider: T) -> Self { + Self { + provider: NoKeyProvider, + token_provider, + } + } +} + +impl PrivacyManager { + /// Borrow the configured token provider or marker capability. + pub fn token_provider(&self) -> &T { + &self.token_provider + } } /// One transformation applied to the source text. @@ -1049,6 +1413,10 @@ pub struct Transformation { pub key_ref: Option, /// Concrete provider version for pseudonymization only. pub resolved_key_version: Option, + /// Provider profile reference for tokenization only. + pub token_ref: Option, + /// Concrete provider profile version for tokenization only. + pub resolved_token_version: Option, } /// Text and audit records produced by a transformation. @@ -1060,6 +1428,34 @@ pub struct TransformResult { pub transformations: Vec, } +/// One token replacement applied while restoring a document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Restoration { + pub source_byte_range: TextRange, + pub source_codepoint_range: TextRange, + pub output_byte_range: TextRange, + pub output_codepoint_range: TextRange, + pub token_ref: String, + pub resolved_token_version: String, +} + +/// Restored text and non-secret range metadata. +#[derive(Clone, PartialEq, Eq)] +pub struct RestoreResult { + pub text: String, + pub restorations: Vec, +} + +impl std::fmt::Debug for RestoreResult { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RestoreResult") + .field("text", &"[REDACTED]") + .field("restorations", &self.restorations) + .finish() + } +} + /// Reason a caller-supplied finding is invalid. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FindingValidationError { @@ -1100,6 +1496,24 @@ pub enum PrivacyErrorCode { InvalidKeyMaterial, /// The provider failed without a more specific safe category. KeyProviderError, + /// Tokenization was selected without a runtime token provider. + TokenProviderRequired, + /// Input contains a malformed canonical token. + InvalidToken, + /// Input uses a canonical token version this core does not support. + UnsupportedTokenVersion, + /// The requested token does not exist. + TokenNotFound, + /// The requested token has expired. + TokenExpired, + /// The token provider denied restoration. + TokenAccessDenied, + /// The token provider returned malformed material. + InvalidTokenMaterial, + /// The token provider is temporarily unavailable. + TokenProviderUnavailable, + /// The token provider failed without a more specific safe category. + TokenProviderError, /// The selected runtime intentionally cannot execute this strategy. UnsupportedStrategy, /// An unexpected non-caller-correctable failure occurred. @@ -1118,6 +1532,15 @@ impl PrivacyErrorCode { Self::KeyProviderUnavailable => "key_provider_unavailable", Self::InvalidKeyMaterial => "invalid_key_material", Self::KeyProviderError => "key_provider_error", + Self::TokenProviderRequired => "token_provider_required", + Self::InvalidToken => "invalid_token", + Self::UnsupportedTokenVersion => "unsupported_token_version", + Self::TokenNotFound => "token_not_found", + Self::TokenExpired => "token_expired", + Self::TokenAccessDenied => "token_access_denied", + Self::InvalidTokenMaterial => "invalid_token_material", + Self::TokenProviderUnavailable => "token_provider_unavailable", + Self::TokenProviderError => "token_provider_error", Self::UnsupportedStrategy => "unsupported_strategy", Self::InternalError => "internal_error", } @@ -1266,6 +1689,69 @@ impl PrivacyError { ) } + fn token_error(code: PrivacyErrorCode, message: &'static str) -> Self { + Self { + code, + reason: None, + path: None, + finding_index: None, + message: message.to_owned(), + } + } + + fn token_provider_required(path: impl Into) -> Self { + Self::key_error( + PrivacyErrorCode::TokenProviderRequired, + path, + "tokenization requires a runtime token provider and request scope", + ) + } + + fn invalid_token() -> Self { + Self::token_error( + PrivacyErrorCode::InvalidToken, + "input contains an invalid token", + ) + } + + fn unsupported_token_version() -> Self { + Self::token_error( + PrivacyErrorCode::UnsupportedTokenVersion, + "input contains an unsupported token version", + ) + } + + fn invalid_token_material() -> Self { + Self::token_error( + PrivacyErrorCode::InvalidTokenMaterial, + "token provider returned invalid token material", + ) + } + + fn from_token_provider_error(error: TokenProviderError) -> Self { + let (code, message) = match error.kind() { + TokenProviderErrorKind::NotFound => { + (PrivacyErrorCode::TokenNotFound, "token was not found") + } + TokenProviderErrorKind::Expired => { + (PrivacyErrorCode::TokenExpired, "token has expired") + } + TokenProviderErrorKind::AccessDenied => ( + PrivacyErrorCode::TokenAccessDenied, + "token access was denied", + ), + TokenProviderErrorKind::Unavailable => ( + PrivacyErrorCode::TokenProviderUnavailable, + "token provider is temporarily unavailable", + ), + TokenProviderErrorKind::ProviderError => ( + PrivacyErrorCode::TokenProviderError, + "token provider could not complete the request", + ), + }; + Self::token_error(code, message) + } + fn internal(message: &'static str) -> Self { Self { code: PrivacyErrorCode::InternalError, @@ -1304,7 +1790,7 @@ impl PrivacyError { Self::key_error( PrivacyErrorCode::UnsupportedStrategy, path, - "the selected runtime does not support pseudonymization", + "the selected runtime does not support this provider-backed strategy", ) } @@ -1417,7 +1903,19 @@ pub fn transform( if let Some(selector) = key_selectors(config, &selected_findings).into_iter().next() { return Err(PrivacyError::provider_required(selector.path)); } - apply_transformations(text, &selected_findings, config, &BTreeMap::new()) + if let Some((_, path)) = tokenization_selections(config, &selected_findings) + .into_iter() + .next() + { + return Err(PrivacyError::token_provider_required(path)); + } + apply_transformations( + text, + &selected_findings, + config, + &BTreeMap::new(), + &BTreeMap::new(), + ) } /// Return the distinct provider keys required after validation, filtering, @@ -1431,41 +1929,382 @@ pub fn required_key_selectors( Ok(key_selectors(config, &selected_findings)) } -/// One resolved key associated with the selector that requested it. -pub struct ResolvedKeyBinding { - selector: KeySelector, - key: ResolvedKey, +/// One resolved key associated with the selector that requested it. +pub struct ResolvedKeyBinding { + selector: KeySelector, + key: ResolvedKey, +} + +impl ResolvedKeyBinding { + /// Associate one provider response with its original selector. + pub fn new(selector: KeySelector, key: ResolvedKey) -> Self { + Self { selector, key } + } +} + +impl std::fmt::Debug for ResolvedKeyBinding { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ResolvedKeyBinding") + .field("selector", &self.selector) + .field("key", &self.key) + .finish() + } +} + +/// Apply a transformation using provider responses already resolved by a thin +/// language binding or another trusted orchestration layer. +pub fn transform_with_resolved_keys( + text: &str, + findings: &[Finding], + config: &TransformationConfig, + resolved_keys: Vec, +) -> Result { + let selected_findings = select_findings(text, findings, config)?; + let selectors = key_selectors(config, &selected_findings); + let resolved_keys = validate_resolved_keys(selectors, resolved_keys)?; + if let Some((_, path)) = tokenization_selections(config, &selected_findings) + .into_iter() + .next() + { + return Err(PrivacyError::token_provider_required(path)); + } + apply_transformations( + text, + &selected_findings, + config, + &resolved_keys, + &BTreeMap::new(), + ) +} + +fn tokenization_selections( + config: &TransformationConfig, + selected_findings: &[Finding], +) -> Vec<(usize, String)> { + selected_findings + .iter() + .enumerate() + .filter_map(|(index, finding)| match config.strategy_for(finding) { + TransformationStrategy::Tokenize(_) => Some((index, config.strategy_path_for(finding))), + _ => None, + }) + .collect() +} + +/// Return the stateful provider items required by a validated transformation. +pub fn required_tokenization_items( + text: &str, + findings: &[Finding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, +) -> Result, PrivacyError> { + let selected = select_findings(text, findings, config)?; + let selections = tokenization_selections(config, &selected); + if selections.is_empty() { + return Ok(Vec::new()); + } + if context.is_none() { + return Err(PrivacyError::token_provider_required( + selections[0].1.clone(), + )); + } + let existing_tokens = parse_tokens_lenient(text); + let mut items = Vec::with_capacity(selections.len()); + for (index, path) in selections { + let finding = &selected[index]; + if existing_tokens.iter().any(|token| { + finding.byte_range.start < token.end && token.start < finding.byte_range.end + }) { + return Err(PrivacyError::key_error( + PrivacyErrorCode::InvalidToken, + path, + "tokenization cannot select an existing canonical token", + )); + } + let TransformationStrategy::Tokenize(tokenize) = config.strategy_for(finding) else { + return Err(PrivacyError::internal( + "tokenization selection changed unexpectedly", + )); + }; + items.push(TokenizeItem { + id: index.to_string(), + exact_value: finding.matched_text.clone(), + token_ref: tokenize.token_ref.clone(), + }); + } + Ok(items) +} + +fn validate_tokenize_results( + items: &[TokenizeItem], + results: Vec, +) -> Result, PrivacyError> { + let expected = items + .iter() + .map(|item| item.id.clone()) + .collect::>(); + let mut validated = BTreeMap::new(); + for result in results { + if !expected.contains(&result.id) + || result.payload.is_empty() + || result.resolved_version.trim().is_empty() + { + return Err(PrivacyError::invalid_token_material()); + } + let item = items + .iter() + .find(|item| item.id == result.id) + .ok_or_else(PrivacyError::invalid_token_material)?; + let envelope = encode_token(&item.token_ref, &result.resolved_version, &result.payload); + if validated + .insert(result.id, (envelope, result.resolved_version)) + .is_some() + { + return Err(PrivacyError::invalid_token_material()); + } + } + if validated.len() != expected.len() { + return Err(PrivacyError::invalid_token_material()); + } + Ok(validated) +} + +/// Complete a transformation using already resolved provider responses. +pub fn transform_with_provider_results( + text: &str, + findings: &[Finding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, + resolved_keys: Vec, + token_results: Vec, +) -> Result { + let selected = select_findings(text, findings, config)?; + let keys = validate_resolved_keys(key_selectors(config, &selected), resolved_keys)?; + let items = required_tokenization_items(text, findings, config, context)?; + let tokens = validate_tokenize_results(&items, token_results)?; + apply_transformations(text, &selected, config, &keys, &tokens) +} + +const TOKEN_PREFIX: &str = "DFTOKENv"; + +#[derive(Clone)] +struct ParsedToken { + start: usize, + end: usize, + token_ref: String, + resolved_version: String, + payload: Vec, + envelope: String, +} + +fn encode_token(token_ref: &str, version: &str, payload: &[u8]) -> String { + let encoder = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let body = format!( + "{}.{}.{}", + encoder.encode(token_ref.as_bytes()), + encoder.encode(version.as_bytes()), + encoder.encode(payload) + ); + format!("DFTOKENv1({}):{body}", body.len()) +} + +fn decode_canonical_component(value: &str) -> Result, PrivacyError> { + if value.is_empty() { + return Err(PrivacyError::invalid_token()); + } + let decoder = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let decoded = decoder + .decode(value) + .map_err(|_| PrivacyError::invalid_token())?; + if decoded.is_empty() || decoder.encode(&decoded) != value { + return Err(PrivacyError::invalid_token()); + } + Ok(decoded) +} + +fn parse_token_at(text: &str, start: usize) -> Result { + let tail = &text[start + TOKEN_PREFIX.len()..]; + let open = tail.find('(').ok_or_else(PrivacyError::invalid_token)?; + let version_text = &tail[..open]; + if version_text.is_empty() || !version_text.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(PrivacyError::invalid_token()); + } + if version_text != "1" { + return Err(PrivacyError::unsupported_token_version()); + } + let after_open = &tail[open + 1..]; + let close = after_open + .find("):") + .ok_or_else(PrivacyError::invalid_token)?; + let length_text = &after_open[..close]; + if length_text.is_empty() || !length_text.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(PrivacyError::invalid_token()); + } + if length_text.len() > 1 && length_text.starts_with('0') { + return Err(PrivacyError::invalid_token()); + } + let body_length = length_text + .bytes() + .try_fold(0usize, |length, digit| { + length + .checked_mul(10)? + .checked_add(usize::from(digit - b'0')) + }) + .ok_or_else(PrivacyError::invalid_token)?; + let body_start = start + TOKEN_PREFIX.len() + open + 1 + close + 2; + let body_end = body_start + .checked_add(body_length) + .ok_or_else(PrivacyError::invalid_token)?; + if body_end > text.len() || !text.is_char_boundary(body_end) { + return Err(PrivacyError::invalid_token()); + } + let body = &text[body_start..body_end]; + let mut components = body.split('.'); + let token_ref_bytes = decode_canonical_component(components.next().unwrap_or_default())?; + let version_bytes = decode_canonical_component(components.next().unwrap_or_default())?; + let payload = decode_canonical_component(components.next().unwrap_or_default())?; + if components.next().is_some() { + return Err(PrivacyError::invalid_token()); + } + let token_ref = + String::from_utf8(token_ref_bytes).map_err(|_| PrivacyError::invalid_token())?; + let resolved_version = + String::from_utf8(version_bytes).map_err(|_| PrivacyError::invalid_token())?; + if token_ref.trim().is_empty() || resolved_version.trim().is_empty() { + return Err(PrivacyError::invalid_token()); + } + Ok(ParsedToken { + start, + end: body_end, + token_ref, + resolved_version, + payload, + envelope: text[start..body_end].to_owned(), + }) +} + +fn parse_tokens(text: &str) -> Result, PrivacyError> { + let mut tokens = Vec::new(); + let mut cursor = 0; + while let Some(relative) = text[cursor..].find(TOKEN_PREFIX) { + let start = cursor + relative; + let token = parse_token_at(text, start)?; + cursor = token.end; + tokens.push(token); + } + Ok(tokens) } -impl ResolvedKeyBinding { - /// Associate one provider response with its original selector. - pub fn new(selector: KeySelector, key: ResolvedKey) -> Self { - Self { selector, key } +fn parse_tokens_lenient(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut cursor = 0; + while let Some(relative) = text[cursor..].find("DFTOKENv1(") { + let start = cursor + relative; + match parse_token_at(text, start) { + Ok(token) => { + cursor = token.end; + tokens.push(token); + } + Err(_) => cursor = start + "DFTOKENv1(".len(), + } } + tokens } -impl std::fmt::Debug for ResolvedKeyBinding { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("ResolvedKeyBinding") - .field("selector", &self.selector) - .field("key", &self.key) - .finish() +/// Parse and deduplicate all canonical tokens before a provider restore call. +pub fn required_restore_items( + text: &str, + _context: &PrivacyContext, +) -> Result, PrivacyError> { + let tokens = parse_tokens(text)?; + let mut ids = BTreeMap::::new(); + let mut items = Vec::new(); + for token in tokens { + if ids.contains_key(&token.envelope) { + continue; + } + let id = items.len().to_string(); + ids.insert(token.envelope, id.clone()); + items.push(RestoreItem { + id, + token_ref: token.token_ref, + resolved_version: token.resolved_version, + payload: token.payload, + }); } + Ok(items) } -/// Apply a transformation using provider responses already resolved by a thin -/// language binding or another trusted orchestration layer. -pub fn transform_with_resolved_keys( +/// Apply complete, validated provider restoration results atomically. +pub fn restore_with_results( text: &str, - findings: &[Finding], - config: &TransformationConfig, - resolved_keys: Vec, -) -> Result { - let selected_findings = select_findings(text, findings, config)?; - let selectors = key_selectors(config, &selected_findings); - let resolved_keys = validate_resolved_keys(selectors, resolved_keys)?; - apply_transformations(text, &selected_findings, config, &resolved_keys) + context: &PrivacyContext, + results: Vec, +) -> Result { + let tokens = parse_tokens(text)?; + let items = required_restore_items(text, context)?; + let expected = items + .iter() + .map(|item| item.id.clone()) + .collect::>(); + let mut values = BTreeMap::new(); + for result in results { + if !expected.contains(&result.id) || values.insert(result.id, result.value).is_some() { + return Err(PrivacyError::invalid_token_material()); + } + } + if values.len() != expected.len() { + return Err(PrivacyError::invalid_token_material()); + } + let mut envelope_ids = BTreeMap::new(); + for token in &tokens { + if !envelope_ids.contains_key(&token.envelope) { + let id = envelope_ids.len().to_string(); + envelope_ids.insert(token.envelope.clone(), id); + } + } + let mut output = String::with_capacity(text.len()); + let mut restorations = Vec::with_capacity(tokens.len()); + let mut cursor = 0; + for token in tokens { + output.push_str(&text[cursor..token.start]); + let output_byte_start = output.len(); + let output_codepoint_start = output.chars().count(); + let id = envelope_ids + .get(&token.envelope) + .ok_or_else(PrivacyError::invalid_token_material)?; + let value = values + .get(id) + .ok_or_else(PrivacyError::invalid_token_material)?; + output.push_str(value); + restorations.push(Restoration { + source_byte_range: TextRange { + start: token.start, + end: token.end, + }, + source_codepoint_range: TextRange { + start: text[..token.start].chars().count(), + end: text[..token.end].chars().count(), + }, + output_byte_range: TextRange { + start: output_byte_start, + end: output.len(), + }, + output_codepoint_range: TextRange { + start: output_codepoint_start, + end: output.chars().count(), + }, + token_ref: token.token_ref, + resolved_token_version: token.resolved_version, + }); + cursor = token.end; + } + output.push_str(&text[cursor..]); + Ok(RestoreResult { + text: output, + restorations, + }) } fn select_findings( @@ -1561,18 +2400,21 @@ fn apply_transformations( selected_findings: &[Finding], config: &TransformationConfig, resolved_keys: &BTreeMap, + tokens: &BTreeMap, ) -> Result { let mut output = String::with_capacity(text.len()); let mut transformations = Vec::with_capacity(selected_findings.len()); let mut source_byte_cursor = 0; - for finding in selected_findings { + for (finding_index, finding) in selected_findings.iter().enumerate() { output.push_str(&text[source_byte_cursor..finding.byte_range.start]); let output_byte_start = output.len(); let output_codepoint_start = output.chars().count(); let strategy = config.strategy_for(finding); let mut key_ref = None; let mut resolved_key_version = None; + let mut token_ref = None; + let mut resolved_token_version = None; let replacement = match strategy { TransformationStrategy::Redact => format!("[{}]", finding.entity_type), TransformationStrategy::Remove => String::new(), @@ -1606,6 +2448,15 @@ fn apply_transformations( resolved_key_version = Some(resolved.resolved_version.clone()); base64::engine::general_purpose::STANDARD.encode(mac.finalize().into_bytes()) } + TransformationStrategy::Tokenize(tokenize) => { + let (envelope, version) = + tokens.get(&finding_index.to_string()).ok_or_else(|| { + PrivacyError::token_provider_required(config.strategy_path_for(finding)) + })?; + token_ref = Some(tokenize.token_ref.clone()); + resolved_token_version = Some(version.clone()); + envelope.clone() + } }; output.push_str(&replacement); @@ -1628,6 +2479,8 @@ fn apply_transformations( }, key_ref, resolved_key_version, + token_ref, + resolved_token_version, }); source_byte_cursor = finding.byte_range.end; } @@ -1639,7 +2492,7 @@ fn apply_transformations( }) } -impl PrivacyManager

{ +impl PrivacyManager { /// Transform caller-supplied findings after atomically resolving every /// distinct key selected by the request. pub async fn transform( @@ -1652,6 +2505,9 @@ impl PrivacyManager

{ let selectors = key_selectors(config, &selected_findings); let mut resolved = BTreeMap::new(); for selector in selectors { + if !self.provider.is_configured() { + return Err(PrivacyError::provider_required(selector.path)); + } let key = self .provider .resolve_key(selector.clone()) @@ -1660,7 +2516,17 @@ impl PrivacyManager

{ validate_resolved_key(&selector, &key)?; resolved.insert(selector.config, key); } - apply_transformations(text, &selected_findings, config, &resolved) + let selections = tokenization_selections(config, &selected_findings); + if let Some((_, path)) = selections.into_iter().next() { + return Err(PrivacyError::token_provider_required(path)); + } + apply_transformations( + text, + &selected_findings, + config, + &resolved, + &BTreeMap::new(), + ) } /// Scan and transform after atomically resolving every selected key. @@ -1679,6 +2545,94 @@ impl PrivacyManager

{ } } +impl PrivacyManager { + /// Transform after resolving keys, then atomically creating every token. + pub async fn transform_with_context( + &self, + text: &str, + findings: &[Finding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, + ) -> Result { + let selected = select_findings(text, findings, config)?; + let mut keys = Vec::new(); + for selector in key_selectors(config, &selected) { + if !self.provider.is_configured() { + return Err(PrivacyError::provider_required(selector.path)); + } + let key = self + .provider + .resolve_key(selector.clone()) + .await + .map_err(|error| PrivacyError::from_provider_error(selector.path.clone(), error))?; + validate_resolved_key(&selector, &key)?; + keys.push(ResolvedKeyBinding::new(selector, key)); + } + let items = required_tokenization_items(text, findings, config, context)?; + let token_results = if items.is_empty() { + Vec::new() + } else { + if !self.token_provider.is_configured() { + let path = tokenization_selections(config, &selected) + .into_iter() + .next() + .map(|(_, path)| path) + .unwrap_or_else(|| "/context/scope".to_owned()); + return Err(PrivacyError::token_provider_required(path)); + } + let scope = context + .ok_or_else(|| PrivacyError::token_provider_required("/context/scope"))? + .scope(); + self.token_provider + .tokenize_batch(scope, items.clone()) + .await + .map_err(PrivacyError::from_token_provider_error)? + }; + transform_with_provider_results(text, findings, config, context, keys, token_results) + } + + /// Scan and transform with request-level provider authorization context. + pub async fn scan_and_transform_with_context( + &self, + text: &str, + config: &ScanAndTransformConfig, + context: Option<&PrivacyContext>, + ) -> Result { + self.transform_with_context( + text, + &scan_with_config(text, config.scan_config()), + config.transformation_config(), + context, + ) + .await + .map_err(|error| error.prefixed("/transform")) + } + + /// Restore every canonical token in a document as one atomic operation. + pub async fn restore( + &self, + text: &str, + context: &PrivacyContext, + ) -> Result { + let items = required_restore_items(text, context)?; + if items.is_empty() { + return Ok(RestoreResult { + text: text.to_owned(), + restorations: Vec::new(), + }); + } + if !self.token_provider.is_configured() { + return Err(PrivacyError::token_provider_required("/restore")); + } + let results = self + .token_provider + .restore_batch(context.scope(), items) + .await + .map_err(PrivacyError::from_token_provider_error)?; + restore_with_results(text, context, results) + } +} + /// Scan text and transform the resulting findings in one explicit convenience operation. pub fn scan_and_transform( text: &str, @@ -2472,21 +3426,25 @@ mod tests { use super::{ Finding, FindingValidationError, KeyProvider, KeyProviderError, KeyProviderErrorKind, KeyProviderFuture, KeySelector, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES, MaskConfig, - MaskConfigError, MaskReveal, PrivacyError, PrivacyErrorCode, PrivacyErrorReason, - PrivacyManager, RegexAllowRule, ResolvedKey, ScanAndTransformConfig, TextRange, - TransformationConfig, TransformationStrategy, parse_scan_and_transform_config, - parse_transformation_config, scan, scan_and_transform, transform, + MaskConfigError, MaskReveal, NoKeyProvider, PrivacyContext, PrivacyError, PrivacyErrorCode, + PrivacyErrorReason, PrivacyManager, RegexAllowRule, ResolvedKey, RestoreProviderFuture, + RestoredValue, ScanAndTransformConfig, TextRange, TokenProvider, TokenProviderError, + TokenProviderErrorKind, TokenizeProviderFuture, TokenizeResult, TransformationConfig, + TransformationStrategy, parse_scan_and_transform_config, parse_transformation_config, + required_restore_items, restore_with_results, scan, scan_and_transform, transform, }; use futures::executor::block_on; use serde_json::json; use std::collections::BTreeMap; - use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; #[derive(Default)] struct TestKeyProvider { responses: BTreeMap<(String, Option), (Vec, String)>, failure: Option, calls: Mutex)>>, + order: Option>>>, } impl TestKeyProvider { @@ -2514,10 +3472,18 @@ mod tests { fn call_count(&self) -> usize { self.calls.lock().unwrap().len() } + + fn with_order(mut self, order: Arc>>) -> Self { + self.order = Some(order); + self + } } impl KeyProvider for TestKeyProvider { fn resolve_key(&self, selector: KeySelector) -> KeyProviderFuture<'_> { + if let Some(order) = &self.order { + order.lock().unwrap().push("key"); + } let identity = ( selector.key_ref().to_owned(), selector.key_version().map(str::to_owned), @@ -2536,6 +3502,91 @@ mod tests { } } + type TokenRecord = (String, String, String, String); + + #[derive(Clone, Default)] + struct TestTokenProvider { + next: Arc, + records: Arc, TokenRecord>>>, + tokenize_calls: Arc>>, + order: Option>>>, + } + + impl TokenProvider for TestTokenProvider { + fn tokenize_batch( + &self, + scope: &str, + items: Vec, + ) -> TokenizeProviderFuture<'_> { + if let Some(order) = &self.order { + order.lock().unwrap().push("token"); + } + let scope = scope.to_owned(); + let records = Arc::clone(&self.records); + let next = Arc::clone(&self.next); + self.tokenize_calls + .lock() + .unwrap() + .push((scope.clone(), items.len())); + Box::pin(async move { + Ok(items + .into_iter() + .map(|item| { + let payload = next.fetch_add(1, Ordering::SeqCst).to_be_bytes().to_vec(); + records.lock().unwrap().insert( + payload.clone(), + ( + scope.clone(), + item.token_ref().to_owned(), + "active-7".to_owned(), + item.exact_value().to_owned(), + ), + ); + TokenizeResult::new(item.id(), payload, "active-7") + }) + .collect()) + }) + } + + fn restore_batch( + &self, + scope: &str, + items: Vec, + ) -> RestoreProviderFuture<'_> { + let scope = scope.to_owned(); + let records = Arc::clone(&self.records); + Box::pin(async move { + let records = records.lock().unwrap(); + items + .into_iter() + .map(|item| { + let Some((bound_scope, token_ref, version, value)) = + records.get(item.payload()) + else { + return Err(TokenProviderError::new(TokenProviderErrorKind::NotFound)); + }; + if bound_scope != &scope + || token_ref != item.token_ref() + || version != item.resolved_version() + { + return Err(TokenProviderError::new( + TokenProviderErrorKind::AccessDenied, + )); + } + Ok(RestoredValue::new(item.id(), value)) + }) + .collect() + }) + } + } + + impl TestTokenProvider { + fn with_order(mut self, order: Arc>>) -> Self { + self.order = Some(order); + self + } + } + fn config(strategy: TransformationStrategy) -> TransformationConfig { TransformationConfig::new(strategy) } @@ -3724,4 +4775,309 @@ mod tests { assert_eq!(result.text, "plain text"); assert!(result.transformations.is_empty()); } + + #[test] + fn tokenization_round_trips_unicode_with_fresh_tokens_and_exact_ranges() { + let text = "👋 jane@example.com jane@example.com"; + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "customers/default"} + })) + .unwrap(); + let context = PrivacyContext::new("tenant/α").unwrap(); + let provider = TestTokenProvider::default(); + let manager = PrivacyManager::::token_provider_only(provider.clone()); + + let transformed = + block_on(manager.transform_with_context(text, &scan(text), &config, Some(&context))) + .unwrap(); + + assert_eq!(transformed.transformations.len(), 2); + assert_ne!( + transformed.transformations[0].replacement, + transformed.transformations[1].replacement, + ); + assert!(transformed.transformations.iter().all(|record| { + record.strategy + == TransformationStrategy::Tokenize( + super::TokenizeConfig::new("customers/default").unwrap(), + ) + && record.token_ref.as_deref() == Some("customers/default") + && record.resolved_token_version.as_deref() == Some("active-7") + && record.key_ref.is_none() + })); + assert_eq!( + provider.tokenize_calls.lock().unwrap().as_slice(), + &[("tenant/α".to_owned(), 2)] + ); + + let restored = block_on(manager.restore(&transformed.text, &context)).unwrap(); + assert_eq!(restored.text, text); + assert_eq!(restored.restorations.len(), 2); + for record in &restored.restorations { + assert!( + transformed.text[record.source_byte_range.start..record.source_byte_range.end] + .starts_with("DFTOKENv1(") + ); + assert_eq!( + &restored.text[record.output_byte_range.start..record.output_byte_range.end], + "jane@example.com", + ); + assert_eq!(record.token_ref, "customers/default"); + assert_eq!(record.resolved_token_version, "active-7"); + } + } + + #[test] + fn restoration_is_atomic_and_scope_bound() { + let text = "jane@example.com"; + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "customers/default"} + })) + .unwrap(); + let context = PrivacyContext::new("tenant-a").unwrap(); + let manager = + PrivacyManager::::token_provider_only(TestTokenProvider::default()); + let transformed = + block_on(manager.transform_with_context(text, &scan(text), &config, Some(&context))) + .unwrap(); + + let error = + block_on(manager.restore(&transformed.text, &PrivacyContext::new("tenant-b").unwrap())) + .unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::TokenAccessDenied); + + let mut tampered = transformed.text.clone(); + let payload_start = tampered.rfind('.').unwrap() + 1; + let replacement = if &tampered[payload_start..payload_start + 1] == "A" { + "B" + } else { + "A" + }; + tampered.replace_range(payload_start..payload_start + 1, replacement); + let error = block_on(manager.restore(&tampered, &context)).unwrap_err(); + assert!(matches!( + error.code(), + PrivacyErrorCode::TokenNotFound | PrivacyErrorCode::InvalidToken + )); + } + + #[test] + fn restore_rejects_malformed_versions_and_incomplete_provider_results() { + let context = PrivacyContext::new("tenant").unwrap(); + assert_eq!( + required_restore_items("DFTOKENv1(999):abc", &context) + .unwrap_err() + .code(), + PrivacyErrorCode::InvalidToken + ); + assert_eq!( + required_restore_items("DFTOKENv1(008):YQ.Yg.Yw", &context) + .unwrap_err() + .code(), + PrivacyErrorCode::InvalidToken + ); + assert_eq!( + required_restore_items("DFTOKENv2(3):abc", &context) + .unwrap_err() + .code(), + PrivacyErrorCode::UnsupportedTokenVersion + ); + let token = super::encode_token("profile", "1", b"payload"); + let error = restore_with_results(&token, &context, Vec::new()).unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::InvalidTokenMaterial); + let ordinary = restore_with_results("ordinary text", &context, Vec::new()).unwrap(); + assert_eq!(ordinary.text, "ordinary text"); + assert!(ordinary.restorations.is_empty()); + } + + #[test] + fn token_provider_responses_require_complete_unique_ids_but_allow_reordering() { + let text = "jane@example.com jane@example.com"; + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "profile"} + })) + .unwrap(); + let findings = scan(text); + let context = PrivacyContext::new("tenant").unwrap(); + + for invalid in [ + Vec::new(), + vec![TokenizeResult::new("unexpected", vec![1], "1")], + vec![ + TokenizeResult::new("0", vec![1], "1"), + TokenizeResult::new("0", vec![2], "1"), + ], + ] { + let error = super::transform_with_provider_results( + text, + &findings, + &config, + Some(&context), + Vec::new(), + invalid, + ) + .unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::InvalidTokenMaterial); + } + + let result = super::transform_with_provider_results( + text, + &findings, + &config, + Some(&context), + Vec::new(), + vec![ + TokenizeResult::new("1", vec![2], "1"), + TokenizeResult::new("0", vec![1], "1"), + ], + ) + .unwrap(); + assert_eq!(result.transformations.len(), 2); + } + + #[test] + fn token_provider_failures_have_sanitized_stable_categories() { + for (kind, expected) in [ + ( + TokenProviderErrorKind::NotFound, + PrivacyErrorCode::TokenNotFound, + ), + ( + TokenProviderErrorKind::Expired, + PrivacyErrorCode::TokenExpired, + ), + ( + TokenProviderErrorKind::AccessDenied, + PrivacyErrorCode::TokenAccessDenied, + ), + ( + TokenProviderErrorKind::Unavailable, + PrivacyErrorCode::TokenProviderUnavailable, + ), + ( + TokenProviderErrorKind::ProviderError, + PrivacyErrorCode::TokenProviderError, + ), + ] { + let error = + super::PrivacyError::from_token_provider_error(TokenProviderError::new(kind)); + assert_eq!(error.code(), expected); + assert!(error.path().is_none()); + } + } + + #[test] + fn mixed_requests_resolve_keys_before_creating_tokens() { + let text = "jane@example.com 2125550100"; + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "profile"}, + "overrides": { + "EMAIL": {"strategy": "pseudonymize", "key_ref": "email-key"} + } + })) + .unwrap(); + let order = Arc::new(Mutex::new(Vec::new())); + let key_provider = TestKeyProvider::default() + .with_key("email-key", None, vec![7; 32], "1") + .with_order(Arc::clone(&order)); + let token_provider = TestTokenProvider::default().with_order(Arc::clone(&order)); + let manager = PrivacyManager::new(key_provider).with_token_provider(token_provider); + let result = block_on(manager.transform_with_context( + text, + &scan(text), + &config, + Some(&PrivacyContext::new("tenant").unwrap()), + )) + .unwrap(); + + assert_eq!(order.lock().unwrap().as_slice(), &["key", "token"]); + assert_eq!(result.transformations.len(), 2); + } + + #[test] + fn token_requests_and_results_redact_sensitive_debug_values() { + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "profile"} + })) + .unwrap(); + let context = PrivacyContext::new("secret-scope").unwrap(); + let items = super::required_tokenization_items( + "jane@example.com", + &scan("jane@example.com"), + &config, + Some(&context), + ) + .unwrap(); + assert!(!format!("{context:?}").contains("secret-scope")); + assert!(!format!("{:?}", items[0]).contains("jane@example.com")); + assert!( + !format!("{:?}", TokenizeResult::new("0", b"secret".to_vec(), "1")).contains("secret") + ); + } + + #[test] + fn token_configuration_is_strict_and_dormant_when_unselected() { + for invalid in [ + json!({"default": {"strategy": "tokenize"}}), + json!({"default": {"strategy": "tokenize", "token_ref": " "}}), + json!({"default": {"strategy": "tokenize", "token_ref": "profile", "ttl": 30}}), + ] { + assert!(parse_transformation_config(&invalid).is_err()); + } + let config = parse_transformation_config(&json!({ + "default": {"strategy": "redact"}, + "overrides": {"PHONE": {"strategy": "tokenize", "token_ref": "profile"}} + })) + .unwrap(); + let result = transform("jane@example.com", &scan("jane@example.com"), &config).unwrap(); + assert_eq!(result.text, "[EMAIL]"); + } + + #[test] + fn optional_manager_capabilities_fail_only_when_selected() { + let context = PrivacyContext::new("tenant").unwrap(); + let manager = PrivacyManager::new(NoKeyProvider); + let ordinary = block_on(manager.restore("ordinary text", &context)).unwrap(); + assert_eq!(ordinary.text, "ordinary text"); + + let config = parse_transformation_config(&json!({ + "default": {"strategy": "tokenize", "token_ref": "profile"} + })) + .unwrap(); + let error = block_on(manager.transform_with_context( + "jane@example.com", + &scan("jane@example.com"), + &config, + Some(&context), + )) + .unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::TokenProviderRequired); + + let token = super::encode_token("profile", "1", b"payload"); + let error = block_on(manager.restore(&token, &context)).unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::TokenProviderRequired); + } + + #[test] + fn tokenization_rejects_nested_tokens_and_restoration_is_not_recursive() { + let context = PrivacyContext::new("tenant").unwrap(); + let inner = super::encode_token("profile", "1", b"inner"); + let finding = supplied_ascii_finding(&inner, "CUSTOM", 0, inner.len(), None, "custom"); + let config = TransformationConfig::new(TransformationStrategy::Tokenize( + super::TokenizeConfig::new("profile").unwrap(), + )); + let error = super::required_tokenization_items(&inner, &[finding], &config, Some(&context)) + .unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::InvalidToken); + + let outer = super::encode_token("profile", "1", b"outer"); + let restored = restore_with_results( + &outer, + &context, + vec![RestoredValue::new("0", inner.clone())], + ) + .unwrap(); + assert_eq!(restored.text, inner); + assert_eq!(restored.restorations.len(), 1); + } } diff --git a/docs/adr/001-privacy-core-contract.md b/docs/adr/001-privacy-core-contract.md index e27aa9d..009b9f7 100644 --- a/docs/adr/001-privacy-core-contract.md +++ b/docs/adr/001-privacy-core-contract.md @@ -52,6 +52,7 @@ with typed enum variants; object-oriented bindings serialize it as: key_ref: "provider-specific-key-reference", key_version?: "provider-specific-version-or-alias" } +{ strategy: "tokenize", token_ref: "provider-profile-reference" } ``` Fields that do not belong to the selected strategy are rejected rather than @@ -354,6 +355,8 @@ Transformation { output_codepoint_range key_ref? // pseudonymize only resolved_key_version? // pseudonymize only + token_ref? // tokenize only + resolved_token_version? // tokenize only } ``` @@ -367,6 +370,8 @@ the source value already possess the input and can use the source ranges. Pseudonymization records include the configured key reference and the concrete version returned by the provider. They never include key material or a plaintext-to-token mapping. Non-pseudonymization records omit both key fields. +Tokenization records include the configured token reference and concrete +provider profile version; other strategies omit both token fields. ### Security meaning of strategies @@ -401,9 +406,11 @@ plaintext-to-token mapping. Non-pseudonymization records omit both key fields. performs no trimming, case folding, Unicode normalization, semantic canonicalization, domain separation, digest truncation, or algorithm negotiation. -- `tokenize` creates opaque reversible or vault-backed tokens. -- `restore` accepts only explicitly reversible tokens and requires - authorization. +- `tokenize` replaces each selected occurrence with an independently issued, + opaque reversible token. It makes no stability guarantee; callers needing + stable equality use `pseudonymize`. +- `restore(text, context={scope})` restores every canonical token in the text + atomically. It has no entity, profile, partial, or ignore-failure filters. Pseudonymization is value pseudonymization, not identity resolution. The core does not infer that different identifiers belong to the same person. @@ -453,6 +460,79 @@ memory. Core ships no cloud-vendor SDK adapter in this slice; AWS, Google, and other integrations belong in separate packages behind the same provider contract. +### Reversible token contract + +Serializable tokenization configuration is exactly `{ strategy: "tokenize", +token_ref }`. `token_ref` is a required, non-secret provider profile selector. +The caller cannot choose algorithms, determinism, TTL, storage, or a concrete +profile version. The provider chooses an active version during tokenization; +Core embeds that returned version so restoration can address historical +versions after rotation. + +Token operations use a separate request context `{ scope }`. Scope is required +only when tokenization is selected and for every restore call. It is an exact, +case-sensitive UTF-8 string; empty and whitespace-only values are rejected, +while Core performs no trimming, case folding, or Unicode normalization. Scope +is not embedded in tokens, returned in results, or placed in errors or debug +output. + +The canonical envelope is: + +```text +DFTOKENv1():.. +``` + +Each body component is canonical unpadded Base64URL. `ref` and `version` +decode to non-empty UTF-8 strings; payload decodes to non-empty opaque bytes. +The decimal byte length uses checked arithmetic and must fit within the +remaining input before decoding. Invalid separators, overflow, padding, empty +components, invalid metadata UTF-8, and truncation reject the whole request. +Unknown envelope versions return `unsupported_token_version`. Core imposes no +arbitrary universal payload or token-count limit; providers may enforce +profile-specific service limits before creating anything. + +The asynchronous `TokenProvider` accepts one tokenization batch per request as +`(scope, [{ id, exact_value, token_ref }])`. Equal plaintext occurrences are +not deduplicated. Each response supplies the matching request-local `id`, +opaque payload bytes, and a non-empty concrete profile version. Missing, +duplicate, unexpected, or malformed responses fail before text mutation. The +provider must bind the exact scope, token reference, and resolved version to +its stored record or authenticated cryptographic payload; envelope metadata is +routing information, not authorization proof. + +Restoration strictly parses the complete text, deduplicates identical +envelopes, and makes one provider call as `(scope, [{ id, token_ref, +resolved_version, payload }])`. The provider returns `{ id, value }`. Core +validates the complete response before replacing anything. Restoration is not +recursive: restored plaintext that resembles a token remains plaintext for +that call. Tokenization rejects selected findings overlapping an existing +valid DataFog token. + +Provider failures use sanitized categories: `token_not_found`, +`token_expired`, `token_access_denied` (including revocation and wrong scope), +`token_provider_unavailable`, and `token_provider_error`. Core additionally +uses `token_provider_required`, `invalid_token`, `unsupported_token_version`, +and `invalid_token_material`. No category includes plaintext, scope, opaque +payload, credentials, or raw provider exception text. Core does not retry, +cache token or plaintext material, log, emit telemetry, or persist audit data. +Providers own retries, idempotency, transaction or staging behavior, cleanup, +lifecycle, and audit. Core guarantees atomic returned output, not rollback of +external entries created before cancellation or failure. + +One `PrivacyManager` composes optional key-provider and token-provider +capabilities. For mixed requests it resolves every pseudonymization key before +stateful token creation and applies text only after all responses validate. +Rust, Python, and Node expose this asynchronous flow. Browser WASM parses +consistently but returns `unsupported_strategy` for selected tokenization and +every restoration call. Core includes no built-in AWS, Google, +database, vault, or cryptographic adapter. + +`RestoreResult` contains restored text and ordered restoration records. Each +record carries token source and restored output byte/code-point ranges, +`token_ref`, and the concrete token profile version. It does not duplicate +restored plaintext or expose payload, scope, credentials, provider topology, +or a token-to-plaintext mapping. + ## Capability-continuity stance Python behavior is classified as preserved, redesigned, compatibility-only, or @@ -478,6 +558,5 @@ and documented separately. This ADR does not choose: -- reversible-token storage or cryptographic construction; - production audit storage; or - custom literal replacement or whitespace-normalizing removal. diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md index 9cc39fc..0779998 100644 --- a/docs/privacy-capability-matrix.md +++ b/docs/privacy-capability-matrix.md @@ -17,8 +17,8 @@ for a PII detection and transformation engine. | Prompt/output guardrails | Out of scope | A governance layer may consume Core findings and results to make and enforce `allow`, `warn`, or `block` decisions. | | Hash replacement | Compatibility only | Exclude unkeyed hashing from canonical Core transformations; consider a plainly named fingerprint in a separate compatibility layer only for an accepted migration requirement. | | Pseudonymization | Redesign | Use HMAC-SHA-256 over exact UTF-8 input with a provider-resolved 256-bit key and full padded-Base64 output. The key defines linkage scope; do not copy numbered Python placeholders. | -| Reversible tokenization | New | Add opaque, authorized, reversible tokens through a key or vault boundary. | -| Restoration | New | Restore only known reversible tokens under explicit authorization and scope checks. | +| Reversible tokenization | New | Issue opaque, versioned tokens through a provider-owned vault or reviewed reversible-crypto boundary, with exact scope/profile binding. | +| Restoration | New | Atomically restore every canonical token through an authorized provider call with exact scope checks and range-only audit metadata. | | Transformation mappings | Redesign | Return ordered transformation records without `matched_text` or a plaintext-to-token mapping. Preserve source ranges and non-sensitive audit metadata. | | Duplicate handling | Redesign | Collapse exact duplicates deterministically. | | Overlap handling | Redesign | Resolve overlaps once in the transformation framework using documented precedence. | diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 226ef63..44e8e5c 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -172,31 +172,55 @@ the original PII. ## Slice 7: Reversible tokenization and restoration -- Define opaque token format and authorization context. -- Introduce a vault or reviewed reversible cryptographic boundary. -- Implement token creation and restoration as one end-to-end slice. -- Reject unknown, expired, wrong-scope, and unauthorized tokens. +**Status: complete** -**Proof:** authorized round trips succeed and every unauthorized variant fails -closed without revealing the original value. +- Add `{ strategy: "tokenize", token_ref }` without algorithm, TTL, + determinism, storage, or caller-pinned version settings. +- Require exact, case-sensitive, non-empty request-level `scope` only for + selected tokenization and every restoration request. +- Keep the provider boundary asynchronous and batched. Tokenization preserves + repeated values as separate items; restoration deduplicates identical + envelopes before the provider call. +- Use the canonical `DFTOKENv1():..` + envelope with unpadded Base64URL components and strict checked parsing. +- Treat token payloads as opaque. Providers own storage or reversible crypto, + authentication, authorization, scope/profile binding, lifecycle, retries, + idempotency, cleanup, and audit logging. +- Resolve every pseudonymization key before stateful token creation, validate + every provider response before mutation, and make Core output atomic. +- Restore every canonical token in the supplied text or return no result. Do + not add filtering, partial restoration, ignore-failure, recursive restore, + or nested tokenization modes. +- Return restoration source/output byte and code-point ranges plus token + reference and concrete profile version, without returning plaintext + mappings, scope, credentials, or opaque payloads. +- Ship full Rust, Python, and Node support. Browser WASM parses the same + configuration and envelope but rejects selected tokenization and every + restoration call with `unsupported_strategy`. +- Ship no built-in cloud, database, vault, or cryptographic provider. + +**Proof:** authorized Unicode and repeated-value round trips preserve exact +ranges while producing independently issued tokens; unauthorized and malformed +variants fail closed without partial Core output or plaintext leakage; nested +tokenization and recursive restoration are absent; Rust, Python, and Node agree +while browser WASM rejects provider-backed work. ## Slice 8: Binding completion and release hardening -Python, Node, and WASM already expose Slices 1 through 4. New stateless -operations should continue to ship through those bindings in the same vertical -slice as their Rust implementation rather than waiting for a separate binding -rollout. +Rust, Python, and Node implement all capabilities through Slice 7. Browser +WASM implements the stateless transformations through Slice 4 and explicitly +rejects provider-backed pseudonymization. New stateless operations should +continue to ship through the bindings in the same vertical slice as their Rust +implementation rather than waiting for a separate binding rollout. Remaining binding work is: 1. add explicitly named UTF-16 code-unit ranges for JavaScript consumers as required by ADR 001, without changing the existing byte or code-point fields; -2. retain Rust, Python, and Node pseudonymization conformance coverage while - keeping browser WASM key handling explicitly unsupported; -3. expose reversible tokenization and restoration only through runtimes with a - separately accepted key-custody, authorization, and storage boundary; and -4. retain installed-package and cross-binding conformance tests as release +2. retain Rust, Python, and Node provider-backed conformance coverage while + keeping browser WASM key and token providers explicitly unsupported; and +3. retain installed-package and cross-binding conformance tests as release gates. Pseudonymization and reversible token storage are not promised in browser WASM diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index d45842a..916a353 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -243,6 +243,55 @@ await assert.rejects( error.path === "/transform/default/key_ref", ); +const tokenRecords = new Map(); +let tokenCounter = 0; +const tokenProvider = { + async tokenizeBatch(scope, items) { + return items.map((item) => { + const payload = Uint8Array.of(++tokenCounter); + tokenRecords.set(payload[0], { + scope, + tokenRef: item.tokenRef, + version: "active-1", + value: item.exactValue, + }); + return { id: item.id, payload, resolvedVersion: "active-1" }; + }); + }, + async restoreBatch(scope, items) { + return items.map((item) => { + const record = tokenRecords.get(item.payload[0]); + if (!record || record.scope !== scope || record.tokenRef !== item.tokenRef || record.version !== item.resolvedVersion) { + const error = new Error("denied"); + error.code = "token_access_denied"; + throw error; + } + return { id: item.id, value: record.value }; + }); + }, +}; +const tokenManager = new PrivacyManager({ tokenProvider }); +const tokenContext = { scope: "tenant/α" }; +const tokenized = await tokenManager.scanAndTransform( + "👋 jane@example.com jane@example.com", + { transform: { default: { strategy: "tokenize", token_ref: "customers/default" } } }, + tokenContext, +); +assert.notEqual(tokenized.transformations[0].replacement, tokenized.transformations[1].replacement); +assert.equal(tokenized.transformations[0].tokenRef, "customers/default"); +assert.equal(tokenized.transformations[0].resolvedTokenVersion, "active-1"); +const restored = await tokenManager.restore(tokenized.text, tokenContext); +assert.equal(restored.text, "👋 jane@example.com jane@example.com"); +assert.equal(restored.restorations.length, 2); +await assert.rejects( + tokenManager.restore(tokenized.text, { scope: "tenant/b" }), + (error) => error instanceof DataFogError && error.code === "token_access_denied", +); +await assert.rejects( + tokenManager.restore("DFTOKENv2(3):abc", tokenContext), + (error) => error instanceof DataFogError && error.code === "unsupported_token_version", +); + assert.throws( () => transform(transformText, scan(transformText), { default: { strategy: "redact" }, diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index 518d8f7..8641048 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -82,6 +82,7 @@ import { init, scan, scanAndTransform, + restore, transform, type EntityType, type Finding, @@ -90,6 +91,7 @@ import { type TextRange, type TransformationConfig, type TransformResult, + type RestoreResult, } from "@datafog/wasm"; const ready: Promise = init(); @@ -115,6 +117,7 @@ const maskConfig: TransformationConfig = { }; const combined: ScanAndTransformConfig = { transform: maskConfig }; const masked: TransformResult = scanAndTransform("Email jane@example.com", combined); +const unchanged: RestoreResult = restore("ordinary text", { scope: "tenant" }); void ready; void entityType; @@ -122,6 +125,7 @@ void range; void transformed; void scannedAndTransformed; void masked; +void unchanged; `.trimStart(), ); @@ -195,7 +199,7 @@ try { await page.goto(serverInfo.url); await page.evaluate(async () => { - const { DataFogError, init, scan, scanAndTransform, transform } = await import( + const { DataFogError, init, restore, scan, scanAndTransform, transform } = await import( "/node_modules/@datafog/wasm/index.js" ); @@ -448,6 +452,35 @@ try { } } + try { + scanAndTransform("Email jane@example.com", { + transform: { + default: { strategy: "tokenize", token_ref: "customers/default" }, + }, + }); + throw new Error("browser tokenization should be unsupported"); + } catch (error) { + if (!(error instanceof DataFogError) || error.code !== "unsupported_strategy") { + throw error; + } + } + try { + restore("ordinary text", { scope: "tenant" }); + throw new Error("browser restoration should be unsupported"); + } catch (error) { + if (!(error instanceof DataFogError) || error.code !== "unsupported_strategy") { + throw error; + } + } + try { + restore("DFTOKENv1(8):YQ.Yg.Yw", { scope: "tenant" }); + throw new Error("browser token restoration should be unsupported"); + } catch (error) { + if (!(error instanceof DataFogError) || error.code !== "unsupported_strategy") { + throw error; + } + } + try { transform(text, findings, { default: { strategy: "redact" },