Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 28 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,21 @@ Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
detectors currently report no confidence score.

The initial transformation strategy is `redact`, which replaces each selected
finding with an unnumbered `[ENTITY_TYPE]` placeholder. `transform` requires
explicit findings; `scan_and_transform` (or `scanAndTransform` in JavaScript)
is the explicit scan-then-transform convenience. Results include the transformed
text and an ordered record for every applied replacement, including its output
byte and code-point ranges.
The initial transformation strategies are `redact`, `mask`, and `remove`.
Redaction uses an unnumbered `[ENTITY_TYPE]` placeholder, masking supports full
or leading/trailing reveal modes, and removal deletes only the exact finding
span. `transform` requires explicit findings; `scan_and_transform` (or
`scanAndTransform` in JavaScript) is the explicit scan-then-transform
convenience. Results include the transformed text and an ordered record for
every applied replacement, including its output byte and code-point ranges.

Object-oriented bindings use a discriminated configuration:

```text
{ strategy: "redact" }
{ strategy: "remove" }
{ strategy: "mask", character: "*", reveal: { direction: "last", count: 4 } }
```

## Packages

Expand Down Expand Up @@ -66,8 +75,14 @@ print(findings[0].entity_type) # EMAIL
print(findings[0].matched_text) # jane@example.com
print(findings[0].byte_range.start) # 6

result = scan_and_transform("Email jane@example.com", "redact")
result = scan_and_transform("Email jane@example.com", {"strategy": "redact"})
assert result.text == "Email [EMAIL]"

masked = scan_and_transform(
"Email jane@example.com",
{"strategy": "mask", "reveal": {"direction": "last", "count": 4}},
)
assert masked.text == "Email ************.com"
```

### Node.js
Expand All @@ -78,7 +93,9 @@ assert result.text == "Email [EMAIL]"
import { scan, scanAndTransform } from "@datafog/node";

console.log(scan("Email jane@example.com"));
console.log(scanAndTransform("Email jane@example.com", "redact").text);
console.log(
scanAndTransform("Email jane@example.com", { strategy: "redact" }).text,
);
```

The release includes prebuilt binaries for macOS (Intel and Apple Silicon), Linux (x64 and ARM64), and Windows x64.
Expand All @@ -92,7 +109,9 @@ import { init, scan, scanAndTransform } from "@datafog/wasm";

await init();
console.log(scan("Email jane@example.com"));
console.log(scanAndTransform("Email jane@example.com", "redact").text);
console.log(
scanAndTransform("Email jane@example.com", { strategy: "redact" }).text,
);
```

## Development
Expand Down
16 changes: 15 additions & 1 deletion bindings/node/dts-header.d.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,18 @@
/** Canonical built-in values are uppercase, but custom detectors may add values. */
export type EntityType = string;

export type TransformationStrategy = "redact";
export type TransformationStrategy = "redact" | "mask" | "remove";

export interface MaskRevealConfig {
readonly direction: "first" | "last";
readonly count: number;
}

export type TransformationConfig =
| { readonly strategy: "redact" }
| { readonly strategy: "remove" }
| {
readonly strategy: "mask";
readonly character?: string;
readonly reveal?: MaskRevealConfig;
};
31 changes: 28 additions & 3 deletions bindings/node/index.d.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,21 @@
/** Canonical built-in values are uppercase, but custom detectors may add values. */
export type EntityType = string;

export type TransformationStrategy = "redact";
export type TransformationStrategy = "redact" | "mask" | "remove";

export interface MaskRevealConfig {
readonly direction: "first" | "last";
readonly count: number;
}

export type TransformationConfig =
| { readonly strategy: "redact" }
| { readonly strategy: "remove" }
| {
readonly strategy: "mask";
readonly character?: string;
readonly reveal?: MaskRevealConfig;
};
export interface Finding {
readonly entityType: EntityType
readonly matchedText: string
Expand All @@ -12,19 +26,30 @@ export interface Finding {
readonly detectorVersion?: string
}

export interface NativeMaskRevealConfig {
direction: string
count: number
}

export interface NativeTransformationConfig {
strategy: TransformationStrategy
character?: string
reveal?: NativeMaskRevealConfig
}

/** Scan text for supported PII findings. */
export declare function scan(text: string): Array<Finding>

/** Scan text and transform the detected findings. */
export declare function scanAndTransform(text: string, strategy: TransformationStrategy): TransformResult
export declare function scanAndTransform(text: string, config: TransformationConfig): TransformResult

export interface TextRange {
readonly start: number
readonly end: number
}

/** Transform explicit findings without scanning implicitly. */
export declare function transform(text: string, findings: Array<Finding>, strategy: TransformationStrategy): TransformResult
export declare function transform(text: string, findings: Array<Finding>, config: TransformationConfig): TransformResult

export interface Transformation {
readonly finding: Finding
Expand Down
72 changes: 62 additions & 10 deletions bindings/node/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,27 +12,79 @@ export function scan(text) {
return nativeScan(text);
}

export function transform(text, findings, strategy) {
function validateConfig(config) {
if (typeof config !== "object" || config === null || Array.isArray(config)) {
throw new TypeError("transformation configuration must be an object");
}
if (!["redact", "mask", "remove"].includes(config.strategy)) {
throw new TypeError("strategy must be 'redact', 'mask', or 'remove'");
}

const allowed =
config.strategy === "mask"
? new Set(["strategy", "character", "reveal"])
: new Set(["strategy"]);
for (const key of Object.keys(config)) {
if (!allowed.has(key)) {
throw new TypeError(`unexpected configuration field: ${key}`);
}
}

if (config.strategy === "mask") {
if (config.character !== undefined) {
if (
typeof config.character !== "string" ||
Array.from(config.character).length !== 1 ||
/[\p{White_Space}\p{Cc}]/u.test(config.character)
) {
throw new TypeError(
"mask character must be one non-whitespace, non-control code point",
);
}
}
if (config.reveal !== undefined) {
if (
typeof config.reveal !== "object" ||
config.reveal === null ||
Array.isArray(config.reveal)
) {
throw new TypeError("mask reveal configuration must be an object");
}
for (const key of Object.keys(config.reveal)) {
if (key !== "direction" && key !== "count") {
throw new TypeError(`unexpected reveal field: ${key}`);
}
}
if (!["first", "last"].includes(config.reveal.direction)) {
throw new TypeError("reveal direction must be 'first' or 'last'");
}
if (
!Number.isSafeInteger(config.reveal.count) ||
config.reveal.count < 0
) {
throw new TypeError("reveal count must be a non-negative safe integer");
}
}
}

return config;
}

export function transform(text, findings, config) {
if (typeof text !== "string") {
throw new TypeError("transform text must be a string");
}
if (!Array.isArray(findings)) {
throw new TypeError("transform findings must be an array");
}
if (typeof strategy !== "string") {
throw new TypeError("transform strategy must be a string");
}

return nativeTransform(text, findings, strategy);
return nativeTransform(text, findings, validateConfig(config));
}

export function scanAndTransform(text, strategy) {
export function scanAndTransform(text, config) {
if (typeof text !== "string") {
throw new TypeError("scanAndTransform text must be a string");
}
if (typeof strategy !== "string") {
throw new TypeError("scanAndTransform strategy must be a string");
}

return nativeScanAndTransform(text, strategy);
return nativeScanAndTransform(text, validateConfig(config));
}
93 changes: 85 additions & 8 deletions bindings/node/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,20 @@ pub struct Finding {
pub detector_version: Option<String>,
}

#[napi(object)]
pub struct NativeMaskRevealConfig {
pub direction: String,
pub count: f64,
}

#[napi(object)]
pub struct NativeTransformationConfig {
#[napi(ts_type = "TransformationStrategy")]
pub strategy: String,
pub character: Option<String>,
pub reveal: Option<NativeMaskRevealConfig>,
}

#[napi(object, object_from_js = false)]
pub struct Transformation {
#[napi(readonly)]
Expand Down Expand Up @@ -109,10 +123,71 @@ fn core_finding(finding: Finding) -> datafog_core::Finding {
}
}

fn core_strategy(strategy: &str) -> napi::Result<datafog_core::TransformationStrategy> {
match strategy {
"redact" => Ok(datafog_core::TransformationStrategy::Redact),
_ => Err(Error::new(Status::InvalidArg, "strategy must be 'redact'")),
fn core_strategy(
config: NativeTransformationConfig,
) -> napi::Result<datafog_core::TransformationStrategy> {
match config.strategy.as_str() {
"redact" if config.character.is_none() && config.reveal.is_none() => {
Ok(datafog_core::TransformationStrategy::Redact)
}
"remove" if config.character.is_none() && config.reveal.is_none() => {
Ok(datafog_core::TransformationStrategy::Remove)
}
"mask" => {
let character = config.character.unwrap_or_else(|| "*".to_owned());
let mut characters = character.chars();
let character = characters
.next()
.filter(|_| characters.next().is_none())
.ok_or_else(|| {
Error::new(
Status::InvalidArg,
"mask character must contain exactly one code point",
)
})?;
let reveal = match config.reveal {
None => datafog_core::MaskReveal::None,
Some(reveal) => {
if !reveal.count.is_finite()
|| reveal.count < 0.0
|| reveal.count.fract() != 0.0
|| reveal.count > 9_007_199_254_740_991.0
{
return Err(Error::new(
Status::InvalidArg,
"reveal count must be a non-negative safe integer",
));
}
let count = reveal.count as usize;
match reveal.direction.as_str() {
"first" => datafog_core::MaskReveal::First(count),
"last" => datafog_core::MaskReveal::Last(count),
_ => {
return Err(Error::new(
Status::InvalidArg,
"reveal direction must be 'first' or 'last'",
));
}
}
}
};
datafog_core::MaskConfig::new(character, reveal)
.map(datafog_core::TransformationStrategy::Mask)
.map_err(|_| {
Error::new(
Status::InvalidArg,
"mask character must not be whitespace or a control character",
)
})
}
"redact" | "remove" => Err(Error::new(
Status::InvalidArg,
"redact and remove do not accept mask configuration",
)),
_ => Err(Error::new(
Status::InvalidArg,
"strategy must be 'redact', 'mask', or 'remove'",
)),
}
}

Expand All @@ -127,6 +202,8 @@ fn js_transform_result(result: datafog_core::TransformResult) -> napi::Result<Tr
finding: js_finding(transformation.finding)?,
strategy: match transformation.strategy {
datafog_core::TransformationStrategy::Redact => "redact".to_owned(),
datafog_core::TransformationStrategy::Remove => "remove".to_owned(),
datafog_core::TransformationStrategy::Mask(_) => "mask".to_owned(),
},
replacement: transformation.replacement,
output_byte_range: js_range(transformation.output_byte_range)?,
Expand All @@ -151,10 +228,10 @@ pub fn scan(text: String) -> napi::Result<Vec<Finding>> {
pub fn transform(
text: String,
findings: Vec<Finding>,
#[napi(ts_arg_type = "TransformationStrategy")] strategy: String,
#[napi(ts_arg_type = "TransformationConfig")] config: NativeTransformationConfig,
) -> napi::Result<TransformResult> {
let findings = findings.into_iter().map(core_finding).collect::<Vec<_>>();
datafog_core::transform(&text, &findings, core_strategy(&strategy)?)
datafog_core::transform(&text, &findings, core_strategy(config)?)
.map_err(|error| Error::new(Status::InvalidArg, error.to_string()))
.and_then(js_transform_result)
}
Expand All @@ -163,9 +240,9 @@ pub fn transform(
#[napi(strict, catch_unwind)]
pub fn scan_and_transform(
text: String,
#[napi(ts_arg_type = "TransformationStrategy")] strategy: String,
#[napi(ts_arg_type = "TransformationConfig")] config: NativeTransformationConfig,
) -> napi::Result<TransformResult> {
datafog_core::scan_and_transform(&text, core_strategy(&strategy)?)
datafog_core::scan_and_transform(&text, core_strategy(config)?)
.map_err(|error| Error::new(Status::GenericFailure, error.to_string()))
.and_then(js_transform_result)
}
Loading
Loading