Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
detectors currently report no confidence score.

The initial transformation strategy is `redact`, which replaces each selected
finding with an unnumbered `[ENTITY_TYPE]` placeholder. `transform` requires
explicit findings; `scan_and_transform` (or `scanAndTransform` in JavaScript)
is the explicit scan-then-transform convenience. Results include the transformed
text and an ordered record for every applied replacement, including its output
byte and code-point ranges.

## Packages

| Runtime | Distribution | Import | Status |
Expand All @@ -31,12 +38,18 @@ cargo add datafog-core
```

```rust
use datafog_core::scan;
use datafog_core::{scan, scan_and_transform, TransformationStrategy};

let findings = scan("Email jane@example.com");
assert_eq!(findings[0].entity_type, "EMAIL");
assert_eq!(findings[0].matched_text, "jane@example.com");
assert_eq!(findings[0].byte_range.start, 6);

let result = scan_and_transform(
"Email jane@example.com",
TransformationStrategy::Redact,
).unwrap();
assert_eq!(result.text, "Email [EMAIL]");
```

### Python
Expand All @@ -46,22 +59,26 @@ python -m pip install datafog-core
```

```python
from datafog_core import scan
from datafog_core import scan, scan_and_transform

findings = scan("Email jane@example.com")
print(findings[0].entity_type) # EMAIL
print(findings[0].matched_text) # jane@example.com
print(findings[0].byte_range.start) # 6

result = scan_and_transform("Email jane@example.com", "redact")
assert result.text == "Email [EMAIL]"
```

### Node.js

`@datafog/node` will install as a native package once its npm release is published.

```js
import { scan } from "@datafog/node";
import { scan, scanAndTransform } from "@datafog/node";

console.log(scan("Email jane@example.com"));
console.log(scanAndTransform("Email jane@example.com", "redact").text);
```

The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64.
Expand All @@ -71,10 +88,11 @@ The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linu
`@datafog/wasm` will install from npm once its first release is published.

```js
import { init, scan } from "@datafog/wasm";
import { init, scan, scanAndTransform } from "@datafog/wasm";

await init();
console.log(scan("Email jane@example.com"));
console.log(scanAndTransform("Email jane@example.com", "redact").text);
```

## Development
Expand Down
2 changes: 2 additions & 0 deletions bindings/node/dts-header.d.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
/** Canonical built-in values are uppercase, but custom detectors may add values. */
export type EntityType = string;

export type TransformationStrategy = "redact";
21 changes: 21 additions & 0 deletions bindings/node/index.d.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
/** Canonical built-in values are uppercase, but custom detectors may add values. */
export type EntityType = string;

export type TransformationStrategy = "redact";
export interface Finding {
readonly entityType: EntityType
readonly matchedText: string
Expand All @@ -13,7 +15,26 @@ export interface Finding {
/** Scan text for supported PII findings. */
export declare function scan(text: string): Array<Finding>

/** Scan text and transform the detected findings. */
export declare function scanAndTransform(text: string, strategy: TransformationStrategy): TransformResult

export interface TextRange {
readonly start: number
readonly end: number
}

/** Transform explicit findings without scanning implicitly. */
export declare function transform(text: string, findings: Array<Finding>, strategy: TransformationStrategy): TransformResult

export interface Transformation {
readonly finding: Finding
readonly strategy: TransformationStrategy
readonly replacement: string
readonly outputByteRange: TextRange
readonly outputCodepointRange: TextRange
}

export interface TransformResult {
readonly text: string
readonly transformations: Array<Transformation>
}
31 changes: 30 additions & 1 deletion bindings/node/index.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { scan as nativeScan } from "./native.js";
import {
scan as nativeScan,
scanAndTransform as nativeScanAndTransform,
transform as nativeTransform,
} from "./native.js";

export function scan(text) {
if (typeof text !== "string") {
Expand All @@ -7,3 +11,28 @@ export function scan(text) {

return nativeScan(text);
}

export function transform(text, findings, strategy) {
if (typeof text !== "string") {
throw new TypeError("transform text must be a string");
}
if (!Array.isArray(findings)) {
throw new TypeError("transform findings must be an array");
}
if (typeof strategy !== "string") {
throw new TypeError("transform strategy must be a string");
}

return nativeTransform(text, findings, strategy);
}

export function scanAndTransform(text, strategy) {
if (typeof text !== "string") {
throw new TypeError("scanAndTransform text must be a string");
}
if (typeof strategy !== "string") {
throw new TypeError("scanAndTransform strategy must be a string");
}

return nativeScanAndTransform(text, strategy);
}
125 changes: 112 additions & 13 deletions bindings/node/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use napi::{Error, Status};
use napi_derive::napi;

#[napi(object, object_from_js = false)]
#[napi(object)]
pub struct TextRange {
#[napi(readonly)]
pub start: u32,
Expand All @@ -12,7 +12,7 @@ pub struct TextRange {
pub end: u32,
}

#[napi(object, object_from_js = false)]
#[napi(object)]
pub struct Finding {
#[napi(readonly, ts_type = "EntityType")]
pub entity_type: String,
Expand All @@ -36,6 +36,33 @@ pub struct Finding {
pub detector_version: Option<String>,
}

#[napi(object, object_from_js = false)]
pub struct Transformation {
#[napi(readonly)]
pub finding: Finding,

#[napi(readonly, ts_type = "TransformationStrategy")]
pub strategy: String,

#[napi(readonly)]
pub replacement: String,

#[napi(readonly)]
pub output_byte_range: TextRange,

#[napi(readonly)]
pub output_codepoint_range: TextRange,
}

#[napi(object, object_from_js = false)]
pub struct TransformResult {
#[napi(readonly)]
pub text: String,

#[napi(readonly)]
pub transformations: Vec<Transformation>,
}

fn js_offset(offset: usize) -> napi::Result<u32> {
u32::try_from(offset).map_err(|_| {
Error::new(
Expand All @@ -52,21 +79,93 @@ fn js_range(range: datafog_core::TextRange) -> napi::Result<TextRange> {
})
}

fn js_finding(finding: datafog_core::Finding) -> napi::Result<Finding> {
Ok(Finding {
entity_type: finding.entity_type,
matched_text: finding.matched_text,
byte_range: js_range(finding.byte_range)?,
codepoint_range: js_range(finding.codepoint_range)?,
confidence: finding.confidence.map(f64::from),
detector_name: finding.detector_name,
detector_version: finding.detector_version,
})
}

fn core_finding(finding: Finding) -> datafog_core::Finding {
datafog_core::Finding {
entity_type: finding.entity_type,
matched_text: finding.matched_text,
byte_range: datafog_core::TextRange {
start: finding.byte_range.start as usize,
end: finding.byte_range.end as usize,
},
codepoint_range: datafog_core::TextRange {
start: finding.codepoint_range.start as usize,
end: finding.codepoint_range.end as usize,
},
confidence: finding.confidence.map(|confidence| confidence as f32),
detector_name: finding.detector_name,
detector_version: finding.detector_version,
}
}

fn core_strategy(strategy: &str) -> napi::Result<datafog_core::TransformationStrategy> {
match strategy {
"redact" => Ok(datafog_core::TransformationStrategy::Redact),
_ => Err(Error::new(Status::InvalidArg, "strategy must be 'redact'")),
}
}

fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result<TransformResult> {
Ok(TransformResult {
text: result.text,
transformations: result
.transformations
.into_iter()
.map(|transformation| {
Ok(Transformation {
finding: js_finding(transformation.finding)?,
strategy: match transformation.strategy {
datafog_core::TransformationStrategy::Redact => "redact".to_owned(),
},
replacement: transformation.replacement,
output_byte_range: js_range(transformation.output_byte_range)?,
output_codepoint_range: js_range(transformation.output_codepoint_range)?,
})
})
.collect::<napi::Result<Vec<_>>>()?,
})
}

/// Scan text for supported PII findings.
#[napi(strict, catch_unwind)]
pub fn scan(text: String) -> napi::Result<Vec<Finding>> {
datafog_core::scan(&text)
.into_iter()
.map(|finding| {
Ok(Finding {
entity_type: finding.entity_type,
matched_text: finding.matched_text,
byte_range: js_range(finding.byte_range)?,
codepoint_range: js_range(finding.codepoint_range)?,
confidence: finding.confidence.map(f64::from),
detector_name: finding.detector_name,
detector_version: finding.detector_version,
})
})
.map(js_finding)
.collect()
}

/// Transform explicit findings without scanning implicitly.
#[napi(strict, catch_unwind)]
pub fn transform(
text: String,
findings: Vec<Finding>,
#[napi(ts_arg_type = "TransformationStrategy")] strategy: String,
) -> napi::Result<TransformResult> {
let findings = findings.into_iter().map(core_finding).collect::<Vec<_>>();
datafog_core::transform(&text, &findings, core_strategy(&strategy)?)
.map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
.and_then(js_transform_result)
}

/// Scan text and transform the detected findings.
#[napi(strict, catch_unwind)]
pub fn scan_and_transform(
text: String,
#[napi(ts_arg_type = "TransformationStrategy")] strategy: String,
) -> napi::Result<TransformResult> {
datafog_core::scan_and_transform(&text, core_strategy(&strategy)?)
.map_err(|error| Error::new(Status::GenericFailure, error.to_string()))
.and_then(js_transform_result)
}
Loading
Loading