Add UTF-16 ranges to JavaScript bindings

README.md

@@ -11,7 +11,9 @@ optional confidence, detector name, optional detector version
 
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
-detectors currently report no confidence score.
+detectors currently report no confidence score. Node.js and browser WASM also
+return an explicitly named UTF-16 code-unit range that can be passed directly
+to JavaScript `String.prototype.slice`.
 
 The transformation strategies are `redact`, `mask`, `remove`, `pseudonymize`,
 and `tokenize` in Rust, Python, and Node.js.
@@ -26,8 +28,9 @@ unsupported in browser WASM.
 `scanAndTransform` in JavaScript) is the explicit scan-then-transform
 convenience. Results include the transformed text and an ordered record for
 every applied replacement, including its source metadata and output byte and
-code-point ranges. Transformation records never include the original matched
-text.
+code-point ranges. Node.js and browser WASM additionally return source and
+output UTF-16 ranges. Transformation records never include the original
+matched text.
 
 Transformation calls require an envelope with a default strategy. It can also
 select entity types, override the strategy per entity, and exempt exact or

bindings/node/dts-header.d.ts

@@ -141,7 +141,7 @@ export declare class PrivacyManager {
   constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
   transform(
     text: string,
-    findings: Finding[],
+    findings: FindingInput[],
     config: TransformationConfig,
     context?: PrivacyContext,
   ): Promise<TransformResult>;

bindings/node/index.d.ts

@@ -141,7 +141,7 @@ export declare class PrivacyManager {
   constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
   transform(
     text: string,
-    findings: Finding[],
+    findings: FindingInput[],
     config: TransformationConfig,
     context?: PrivacyContext,
   ): Promise<TransformResult>;
@@ -157,11 +157,22 @@ export interface Finding {
   readonly matchedText: string
   readonly byteRange: TextRange
   readonly codepointRange: TextRange
+  readonly utf16Range: TextRange
   readonly confidence?: number
   readonly detectorName: string
   readonly detectorVersion?: string
 }
 
+export interface FindingInput {
+  entityType: EntityType
+  matchedText: string
+  byteRange: TextRange
+  codepointRange: TextRange
+  confidence?: number
+  detectorName: string
+  detectorVersion?: string
+}
+
 export interface KeySelector {
   readonly index: number
   readonly keyRef: string
@@ -176,11 +187,11 @@ export interface PreparedScanAndTransform {
 
 export declare function prepareScanAndTransform(text: string, config: ScanAndTransformConfig): PreparedScanAndTransform
 
-export declare function requiredKeySelectors(text: string, findings: Array<Finding>, config: TransformationConfig): Array<KeySelector>
+export declare function requiredKeySelectors(text: string, findings: FindingInput[], config: TransformationConfig): Array<KeySelector>
 
 export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array<RestoreItem>
 
-export declare function requiredTokenizationItems(text: string, findings: Array<Finding>, config: TransformationConfig, context?: PrivacyContext | undefined): Array<TokenizeItem>
+export declare function requiredTokenizationItems(text: string, findings: FindingInput[], config: TransformationConfig, context?: PrivacyContext | undefined): Array<TokenizeItem>
 
 export interface ResolvedKeyInput {
   selectorIndex: number
@@ -191,8 +202,10 @@ export interface ResolvedKeyInput {
 export interface Restoration {
   readonly sourceByteRange: TextRange
   readonly sourceCodepointRange: TextRange
+  readonly sourceUtf16Range: TextRange
   readonly outputByteRange: TextRange
   readonly outputCodepointRange: TextRange
+  readonly outputUtf16Range: TextRange
   readonly tokenRef: string
   readonly resolvedTokenVersion: string
 }
@@ -240,19 +253,21 @@ export interface TokenizeResultInput {
 }
 
 /** Transform explicit findings without scanning implicitly. */
-export declare function transform(text: string, findings: Array<Finding>, config: TransformationConfig): TransformResult
+export declare function transform(text: string, findings: FindingInput[], config: TransformationConfig): TransformResult
 
 export interface Transformation {
   readonly entityType: string
   readonly sourceByteRange: TextRange
   readonly sourceCodepointRange: TextRange
+  readonly sourceUtf16Range: TextRange
   readonly confidence?: number
   readonly detectorName: string
   readonly detectorVersion?: string
   readonly strategy: TransformationStrategy
   readonly replacement: string
   readonly outputByteRange: TextRange
   readonly outputCodepointRange: TextRange
+  readonly outputUtf16Range: TextRange
   readonly keyRef?: string
   readonly resolvedKeyVersion?: string
   readonly tokenRef?: string
@@ -264,6 +279,6 @@ export interface TransformResult {
   readonly transformations: Array<Transformation>
 }
 
-export declare function transformWithProviderResults(text: string, findings: Array<Finding>, config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array<ResolvedKeyInput>, tokenResults: Array<TokenizeResultInput>): TransformResult
+export declare function transformWithProviderResults(text: string, findings: FindingInput[], config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array<ResolvedKeyInput>, tokenResults: Array<TokenizeResultInput>): TransformResult
 
-export declare function transformWithResolvedKeys(text: string, findings: Array<Finding>, config: TransformationConfig, resolvedKeys: Array<ResolvedKeyInput>): TransformResult
+export declare function transformWithResolvedKeys(text: string, findings: FindingInput[], config: TransformationConfig, resolvedKeys: Array<ResolvedKeyInput>): TransformResult

bindings/node/src/lib.rs

@@ -27,6 +27,9 @@ pub struct Finding {
     #[napi(readonly)]
     pub codepoint_range: TextRange,
 
+    #[napi(readonly)]
+    pub utf16_range: TextRange,
+
     #[napi(readonly)]
     pub confidence: Option<f64>,
 
@@ -37,6 +40,18 @@ pub struct Finding {
     pub detector_version: Option<String>,
 }
 
+#[napi(object, object_to_js = false)]
+pub struct FindingInput {
+    #[napi(ts_type = "EntityType")]
+    pub entity_type: String,
+    pub matched_text: String,
+    pub byte_range: TextRange,
+    pub codepoint_range: TextRange,
+    pub confidence: Option<f64>,
+    pub detector_name: String,
+    pub detector_version: Option<String>,
+}
+
 #[napi(object, object_from_js = false)]
 pub struct Transformation {
     #[napi(readonly)]
@@ -48,6 +63,9 @@ pub struct Transformation {
     #[napi(readonly)]
     pub source_codepoint_range: TextRange,
 
+    #[napi(readonly)]
+    pub source_utf16_range: TextRange,
+
     #[napi(readonly)]
     pub confidence: Option<f64>,
 
@@ -69,6 +87,9 @@ pub struct Transformation {
     #[napi(readonly)]
     pub output_codepoint_range: TextRange,
 
+    #[napi(readonly)]
+    pub output_utf16_range: TextRange,
+
     #[napi(readonly)]
     pub key_ref: Option<String>,
 
@@ -157,10 +178,14 @@ pub struct Restoration {
     #[napi(readonly)]
     pub source_codepoint_range: TextRange,
     #[napi(readonly)]
+    pub source_utf16_range: TextRange,
+    #[napi(readonly)]
     pub output_byte_range: TextRange,
     #[napi(readonly)]
     pub output_codepoint_range: TextRange,
     #[napi(readonly)]
+    pub output_utf16_range: TextRange,
+    #[napi(readonly)]
     pub token_ref: String,
     #[napi(readonly)]
     pub resolved_token_version: String,
@@ -199,19 +224,26 @@ fn js_range(range: datafog_core::TextRange) -> napi::Result<TextRange> {
     })
 }
 
-fn js_finding(finding: datafog_core::Finding) -> napi::Result<Finding> {
+fn js_utf16_range(text: &str, range: datafog_core::TextRange) -> napi::Result<TextRange> {
+    datafog_core::utf16_range(text, range)
+        .map_err(|error| Error::new(Status::GenericFailure, error.to_string()))
+        .and_then(js_range)
+}
+
+fn js_finding(text: &str, finding: datafog_core::Finding) -> napi::Result<Finding> {
     Ok(Finding {
         entity_type: finding.entity_type,
         matched_text: finding.matched_text,
         byte_range: js_range(finding.byte_range)?,
         codepoint_range: js_range(finding.codepoint_range)?,
+        utf16_range: js_utf16_range(text, finding.byte_range)?,
         confidence: finding.confidence.map(f64::from),
         detector_name: finding.detector_name,
         detector_version: finding.detector_version,
     })
 }
 
-fn core_finding(finding: Finding) -> datafog_core::Finding {
+fn core_finding(finding: FindingInput) -> datafog_core::Finding {
     datafog_core::Finding {
         entity_type: finding.entity_type,
         matched_text: finding.matched_text,
@@ -246,9 +278,12 @@ fn js_privacy_error(error: datafog_core::PrivacyError) -> Error {
     Error::new(status, payload.to_string())
 }
 
-fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result<TransformResult> {
+fn js_transform_result(
+    source_text: &str,
+    result: datafog_core::TransformResult,
+) -> napi::Result<TransformResult> {
+    let output_text = &result.text;
     Ok(TransformResult {
-        text: result.text,
         transformations: result
             .transformations
             .into_iter()
@@ -257,6 +292,10 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result<Tr
                     entity_type: transformation.entity_type,
                     source_byte_range: js_range(transformation.source_byte_range)?,
                     source_codepoint_range: js_range(transformation.source_codepoint_range)?,
+                    source_utf16_range: js_utf16_range(
+                        source_text,
+                        transformation.source_byte_range,
+                    )?,
                     confidence: transformation.confidence.map(f64::from),
                     detector_name: transformation.detector_name,
                     detector_version: transformation.detector_version,
@@ -272,13 +311,18 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result<Tr
                     replacement: transformation.replacement,
                     output_byte_range: js_range(transformation.output_byte_range)?,
                     output_codepoint_range: js_range(transformation.output_codepoint_range)?,
+                    output_utf16_range: js_utf16_range(
+                        output_text,
+                        transformation.output_byte_range,
+                    )?,
                     key_ref: transformation.key_ref,
                     resolved_key_version: transformation.resolved_key_version,
                     token_ref: transformation.token_ref,
                     resolved_token_version: transformation.resolved_token_version,
                 })
             })
             .collect::<napi::Result<Vec<_>>>()?,
+        text: result.text,
     })
 }
 
@@ -295,23 +339,29 @@ fn core_token_results(results: Vec<TokenizeResultInput>) -> Vec<datafog_core::To
         .collect()
 }
 
-fn js_restore_result(result: datafog_core::RestoreResult) -> napi::Result<RestoreResult> {
+fn js_restore_result(
+    source_text: &str,
+    result: datafog_core::RestoreResult,
+) -> napi::Result<RestoreResult> {
+    let output_text = &result.text;
     Ok(RestoreResult {
-        text: result.text,
         restorations: result
             .restorations
             .into_iter()
             .map(|record| {
                 Ok(Restoration {
                     source_byte_range: js_range(record.source_byte_range)?,
                     source_codepoint_range: js_range(record.source_codepoint_range)?,
+                    source_utf16_range: js_utf16_range(source_text, record.source_byte_range)?,
                     output_byte_range: js_range(record.output_byte_range)?,
                     output_codepoint_range: js_range(record.output_codepoint_range)?,
+                    output_utf16_range: js_utf16_range(output_text, record.output_byte_range)?,
                     token_ref: record.token_ref,
                     resolved_token_version: record.resolved_token_version,
                 })
             })
             .collect::<napi::Result<Vec<_>>>()?,
+        text: result.text,
     })
 }
 
@@ -366,7 +416,7 @@ pub fn scan(
     };
     datafog_core::scan_with_config(&text, &config)
         .into_iter()
-        .map(js_finding)
+        .map(|finding| js_finding(&text, finding))
         .collect()
 }
 
@@ -375,15 +425,15 @@ pub fn scan(
 pub fn transform(
     env: Env,
     text: String,
-    findings: Vec<Finding>,
+    #[napi(ts_arg_type = "FindingInput[]")] findings: Vec<FindingInput>,
     #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>,
 ) -> napi::Result<TransformResult> {
     let config: serde_json::Value = env.from_js_value(config)?;
     let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?;
     let findings = findings.into_iter().map(core_finding).collect::<Vec<_>>();
     datafog_core::transform(&text, &findings, &config)
         .map_err(js_privacy_error)
-        .and_then(js_transform_result)
+        .and_then(|result| js_transform_result(&text, result))
 }
 
 /// Scan text and transform the detected findings.
@@ -398,14 +448,14 @@ pub fn scan_and_transform(
         datafog_core::parse_scan_and_transform_config(&config).map_err(js_privacy_error)?;
     datafog_core::scan_and_transform(&text, &config)
         .map_err(js_privacy_error)
-        .and_then(js_transform_result)
+        .and_then(|result| js_transform_result(&text, result))
 }
 
 #[napi(strict, catch_unwind)]
 pub fn required_key_selectors(
     env: Env,
     text: String,
-    findings: Vec<Finding>,
+    #[napi(ts_arg_type = "FindingInput[]")] findings: Vec<FindingInput>,
     #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>,
 ) -> napi::Result<Vec<KeySelector>> {
     let config: serde_json::Value = env.from_js_value(config)?;
@@ -420,7 +470,7 @@ pub fn required_key_selectors(
 pub fn transform_with_resolved_keys(
     env: Env,
     text: String,
-    findings: Vec<Finding>,
+    #[napi(ts_arg_type = "FindingInput[]")] findings: Vec<FindingInput>,
     #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>,
     resolved_keys: Vec<ResolvedKeyInput>,
 ) -> napi::Result<TransformResult> {
@@ -432,7 +482,7 @@ pub fn transform_with_resolved_keys(
     let bindings = core_key_bindings(selectors, resolved_keys)?;
     datafog_core::transform_with_resolved_keys(&text, &findings, &config, bindings)
         .map_err(js_privacy_error)
-        .and_then(js_transform_result)
+        .and_then(|result| js_transform_result(&text, result))
 }
 
 #[napi(strict, catch_unwind)]
@@ -451,7 +501,7 @@ pub fn prepare_scan_and_transform(
     Ok(PreparedScanAndTransform {
         findings: findings
             .into_iter()
-            .map(js_finding)
+            .map(|finding| js_finding(&text, finding))
             .collect::<napi::Result<Vec<_>>>()?,
         selectors: js_key_selectors(&selectors)?,
     })
@@ -461,7 +511,7 @@ pub fn prepare_scan_and_transform(
 pub fn required_tokenization_items(
     env: Env,
     text: String,
-    findings: Vec<Finding>,
+    #[napi(ts_arg_type = "FindingInput[]")] findings: Vec<FindingInput>,
     #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>,
     #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option<Unknown<'_>>,
 ) -> napi::Result<Vec<TokenizeItem>> {
@@ -493,7 +543,7 @@ pub fn required_tokenization_items(
 pub fn transform_with_provider_results(
     env: Env,
     text: String,
-    findings: Vec<Finding>,
+    #[napi(ts_arg_type = "FindingInput[]")] findings: Vec<FindingInput>,
     #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>,
     #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option<Unknown<'_>>,
     resolved_keys: Vec<ResolvedKeyInput>,
@@ -521,7 +571,7 @@ pub fn transform_with_provider_results(
         core_token_results(token_results),
     )
     .map_err(js_privacy_error)
-    .and_then(js_transform_result)
+    .and_then(|result| js_transform_result(&text, result))
 }
 
 #[napi(strict, catch_unwind)]
@@ -562,5 +612,5 @@ pub fn restore_with_results(
         .collect();
     datafog_core::restore_with_results(&text, &context, results)
         .map_err(js_privacy_error)
-        .and_then(js_restore_result)
+        .and_then(|result| js_restore_result(&text, result))
 }