Implement Slice 7 reversible tokenization and restoration

README.md

@@ -13,12 +13,15 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score.
 
-The transformation strategies are `redact`, `mask`, `remove`, and
-`pseudonymize` in Rust, Python, and Node.js.
+The transformation strategies are `redact`, `mask`, `remove`, `pseudonymize`,
+and `tokenize` in Rust, Python, and Node.js.
 Redaction uses an unnumbered `[ENTITY_TYPE]` placeholder, masking supports full
 or leading/trailing reveal modes, and removal deletes only the exact finding
 span. Pseudonymization uses provider-resolved 256-bit keys and deterministic
-HMAC-SHA-256 tokens; it is deliberately unsupported in browser WASM.
+HMAC-SHA-256 tokens. Tokenization uses an application-supplied asynchronous
+provider to issue opaque `DFTOKENv1(...)` envelopes and restore them under an
+exact request-level scope. Both provider-backed strategies are deliberately
+unsupported in browser WASM.
 `transform` requires explicit findings; `scan_and_transform` (or
 `scanAndTransform` in JavaScript) is the explicit scan-then-transform
 convenience. Results include the transformed text and an ordered record for
@@ -134,6 +137,17 @@ async def pseudonymize():
     )
 
 pseudonymized = asyncio.run(pseudonymize())
+
+# A token provider implements tokenize_batch(scope, items) and
+# restore_batch(scope, items). It owns storage or reversible cryptography,
+# authorization, lifecycle, and audit.
+token_manager = PrivacyManager(None, token_provider=TokenProvider())
+tokenized = asyncio.run(token_manager.scan_and_transform(
+    "Email jane@example.com",
+    {"transform": {"default": {"strategy": "tokenize", "token_ref": "customers/default"}}},
+    {"scope": "tenant-a"},
+))
+restored = asyncio.run(token_manager.restore(tokenized.text, {"scope": "tenant-a"}))
 ```
 
 ### Node.js
@@ -160,6 +174,14 @@ const pseudonymized = await manager.scanAndTransform("Email jane@example.com", {
     default: { strategy: "pseudonymize", key_ref: "customers/email" },
   },
 });
+
+const tokenManager = new PrivacyManager({ tokenProvider });
+const tokenized = await tokenManager.scanAndTransform(
+  "Email jane@example.com",
+  { transform: { default: { strategy: "tokenize", token_ref: "customers/default" } } },
+  { scope: "tenant-a" },
+);
+const restored = await tokenManager.restore(tokenized.text, { scope: "tenant-a" });
 ```
 
 The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64.

bindings/node/dts-header.d.ts

@@ -5,7 +5,8 @@ export type TransformationStrategy =
   | "redact"
   | "mask"
   | "remove"
-  | "pseudonymize";
+  | "pseudonymize"
+  | "tokenize";
 
 export interface MaskRevealConfig {
   readonly direction: "first" | "last";
@@ -20,6 +21,10 @@ export type TransformationStrategyConfig =
       readonly key_ref: string;
       readonly key_version?: string;
     }
+  | {
+      readonly strategy: "tokenize";
+      readonly token_ref: string;
+    }
   | {
       readonly strategy: "mask";
       readonly character?: string;
@@ -61,6 +66,15 @@ export type DataFogErrorCode =
   | "key_provider_unavailable"
   | "invalid_key_material"
   | "key_provider_error"
+  | "token_provider_required"
+  | "invalid_token"
+  | "unsupported_token_version"
+  | "token_not_found"
+  | "token_expired"
+  | "token_access_denied"
+  | "invalid_token_material"
+  | "token_provider_unavailable"
+  | "token_provider_error"
   | "unsupported_strategy"
   | "internal_error";
 
@@ -85,15 +99,56 @@ export interface KeyProvider {
   resolveKey(request: KeyProviderRequest): Promise<KeyProviderResponse>;
 }
 
+export interface PrivacyContext {
+  readonly scope: string;
+}
+
+export interface TokenizeProviderItem {
+  readonly id: string;
+  readonly exactValue: string;
+  readonly tokenRef: string;
+}
+
+export interface TokenizeProviderResult {
+  readonly id: string;
+  readonly payload: Uint8Array;
+  readonly resolvedVersion: string;
+}
+
+export interface RestoreProviderItem {
+  readonly id: string;
+  readonly tokenRef: string;
+  readonly resolvedVersion: string;
+  readonly payload: Uint8Array;
+}
+
+export interface RestoreProviderResult {
+  readonly id: string;
+  readonly value: string;
+}
+
+export interface TokenProvider {
+  tokenizeBatch(scope: string, items: TokenizeProviderItem[]): Promise<TokenizeProviderResult[]>;
+  restoreBatch(scope: string, items: RestoreProviderItem[]): Promise<RestoreProviderResult[]>;
+}
+
+export interface PrivacyManagerProviders {
+  readonly keyProvider?: KeyProvider;
+  readonly tokenProvider?: TokenProvider;
+}
+
 export declare class PrivacyManager {
-  constructor(provider: KeyProvider);
+  constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
   transform(
     text: string,
     findings: Finding[],
     config: TransformationConfig,
+    context?: PrivacyContext,
   ): Promise<TransformResult>;
   scanAndTransform(
     text: string,
     config: ScanAndTransformConfig,
+    context?: PrivacyContext,
   ): Promise<TransformResult>;
+  restore(text: string, context: PrivacyContext): Promise<RestoreResult>;
 }

bindings/node/index.d.ts

@@ -5,7 +5,8 @@ export type TransformationStrategy =
   | "redact"
   | "mask"
   | "remove"
-  | "pseudonymize";
+  | "pseudonymize"
+  | "tokenize";
 
 export interface MaskRevealConfig {
   readonly direction: "first" | "last";
@@ -20,6 +21,10 @@ export type TransformationStrategyConfig =
       readonly key_ref: string;
       readonly key_version?: string;
     }
+  | {
+      readonly strategy: "tokenize";
+      readonly token_ref: string;
+    }
   | {
       readonly strategy: "mask";
       readonly character?: string;
@@ -61,6 +66,15 @@ export type DataFogErrorCode =
   | "key_provider_unavailable"
   | "invalid_key_material"
   | "key_provider_error"
+  | "token_provider_required"
+  | "invalid_token"
+  | "unsupported_token_version"
+  | "token_not_found"
+  | "token_expired"
+  | "token_access_denied"
+  | "invalid_token_material"
+  | "token_provider_unavailable"
+  | "token_provider_error"
   | "unsupported_strategy"
   | "internal_error";
 
@@ -85,17 +99,58 @@ export interface KeyProvider {
   resolveKey(request: KeyProviderRequest): Promise<KeyProviderResponse>;
 }
 
+export interface PrivacyContext {
+  readonly scope: string;
+}
+
+export interface TokenizeProviderItem {
+  readonly id: string;
+  readonly exactValue: string;
+  readonly tokenRef: string;
+}
+
+export interface TokenizeProviderResult {
+  readonly id: string;
+  readonly payload: Uint8Array;
+  readonly resolvedVersion: string;
+}
+
+export interface RestoreProviderItem {
+  readonly id: string;
+  readonly tokenRef: string;
+  readonly resolvedVersion: string;
+  readonly payload: Uint8Array;
+}
+
+export interface RestoreProviderResult {
+  readonly id: string;
+  readonly value: string;
+}
+
+export interface TokenProvider {
+  tokenizeBatch(scope: string, items: TokenizeProviderItem[]): Promise<TokenizeProviderResult[]>;
+  restoreBatch(scope: string, items: RestoreProviderItem[]): Promise<RestoreProviderResult[]>;
+}
+
+export interface PrivacyManagerProviders {
+  readonly keyProvider?: KeyProvider;
+  readonly tokenProvider?: TokenProvider;
+}
+
 export declare class PrivacyManager {
-  constructor(provider: KeyProvider);
+  constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
   transform(
     text: string,
     findings: Finding[],
     config: TransformationConfig,
+    context?: PrivacyContext,
   ): Promise<TransformResult>;
   scanAndTransform(
     text: string,
     config: ScanAndTransformConfig,
+    context?: PrivacyContext,
   ): Promise<TransformResult>;
+  restore(text: string, context: PrivacyContext): Promise<RestoreResult>;
 }
 export interface Finding {
   readonly entityType: EntityType
@@ -123,12 +178,44 @@ export declare function prepareScanAndTransform(text: string, config: ScanAndTra
 
 export declare function requiredKeySelectors(text: string, findings: Array<Finding>, config: TransformationConfig): Array<KeySelector>
 
+export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array<RestoreItem>
+
+export declare function requiredTokenizationItems(text: string, findings: Array<Finding>, config: TransformationConfig, context?: PrivacyContext | undefined): Array<TokenizeItem>
+
 export interface ResolvedKeyInput {
   selectorIndex: number
   key: Uint8Array
   resolvedVersion: string
 }
 
+export interface Restoration {
+  readonly sourceByteRange: TextRange
+  readonly sourceCodepointRange: TextRange
+  readonly outputByteRange: TextRange
+  readonly outputCodepointRange: TextRange
+  readonly tokenRef: string
+  readonly resolvedTokenVersion: string
+}
+
+export interface RestoredValueInput {
+  id: string
+  value: string
+}
+
+export interface RestoreItem {
+  readonly id: string
+  readonly tokenRef: string
+  readonly resolvedVersion: string
+  readonly payload: Uint8Array
+}
+
+export interface RestoreResult {
+  readonly text: string
+  readonly restorations: Array<Restoration>
+}
+
+export declare function restoreWithResults(text: string, context: PrivacyContext, results: Array<RestoredValueInput>): RestoreResult
+
 /** Scan text for supported PII findings. */
 export declare function scan(text: string, config?: ScanConfig | undefined): Array<Finding>
 
@@ -140,6 +227,18 @@ export interface TextRange {
   readonly end: number
 }
 
+export interface TokenizeItem {
+  readonly id: string
+  readonly exactValue: string
+  readonly tokenRef: string
+}
+
+export interface TokenizeResultInput {
+  id: string
+  payload: Uint8Array
+  resolvedVersion: string
+}
+
 /** Transform explicit findings without scanning implicitly. */
 export declare function transform(text: string, findings: Array<Finding>, config: TransformationConfig): TransformResult
 
@@ -156,11 +255,15 @@ export interface Transformation {
   readonly outputCodepointRange: TextRange
   readonly keyRef?: string
   readonly resolvedKeyVersion?: string
+  readonly tokenRef?: string
+  readonly resolvedTokenVersion?: string
 }
 
 export interface TransformResult {
   readonly text: string
   readonly transformations: Array<Transformation>
 }
 
+export declare function transformWithProviderResults(text: string, findings: Array<Finding>, config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array<ResolvedKeyInput>, tokenResults: Array<TokenizeResultInput>): TransformResult
+
 export declare function transformWithResolvedKeys(text: string, findings: Array<Finding>, config: TransformationConfig, resolvedKeys: Array<ResolvedKeyInput>): TransformResult