Implement Slice One finding and scan contract

README.md

@@ -2,13 +2,16 @@
 
 Fast structured PII detection, implemented in Rust and exposed for Rust, Python, Node.js, and browsers.
 
-It detects `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, and `ZIP_CODE`. Every binding returns the same entity shape:
+It detects `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, and `ZIP_CODE`. Every binding returns the same finding information:
 
 ```text
-label, text, start, end
+entity type, matched text, byte range, code-point range,
+optional confidence, detector name, optional detector version
 ```
 
-`start` and `end` are zero-based Unicode code-point offsets; `end` is exclusive.
+Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
+UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
+detectors currently report no confidence score.
 
 ## Packages
 
@@ -30,9 +33,10 @@ cargo add datafog-core
 ```rust
 use datafog_core::scan;
 
-let entities = scan("Email jane@example.com");
-assert_eq!(entities[0].label, "EMAIL");
-assert_eq!(entities[0].text, "jane@example.com");
+let findings = scan("Email jane@example.com");
+assert_eq!(findings[0].entity_type, "EMAIL");
+assert_eq!(findings[0].matched_text, "jane@example.com");
+assert_eq!(findings[0].byte_range.start, 6);
 ```
 
 ### Python
@@ -44,9 +48,10 @@ python -m pip install datafog-core
 ```python
 from datafog_core import scan
 
-entities = scan("Email jane@example.com")
-print(entities[0].label)  # EMAIL
-print(entities[0].text)   # jane@example.com
+findings = scan("Email jane@example.com")
+print(findings[0].entity_type)       # EMAIL
+print(findings[0].matched_text)      # jane@example.com
+print(findings[0].byte_range.start)  # 6
 ```
 
 ### Node.js

bindings/node/dts-header.d.ts

@@ -1,8 +1,2 @@
-export type Label =
-  | "EMAIL"
-  | "PHONE"
-  | "SSN"
-  | "CREDIT_CARD"
-  | "IP_ADDRESS"
-  | "DATE"
-  | "ZIP_CODE";
+/** Canonical built-in values are uppercase, but custom detectors may add values. */
+export type EntityType = string;

bindings/node/index.d.ts

@@ -1,17 +1,19 @@
-export type Label =
-  | "EMAIL"
-  | "PHONE"
-  | "SSN"
-  | "CREDIT_CARD"
-  | "IP_ADDRESS"
-  | "DATE"
-  | "ZIP_CODE";
-export interface Entity {
-  readonly label: Label
-  readonly text: string
+/** Canonical built-in values are uppercase, but custom detectors may add values. */
+export type EntityType = string;
+export interface Finding {
+  readonly entityType: EntityType
+  readonly matchedText: string
+  readonly byteRange: TextRange
+  readonly codepointRange: TextRange
+  readonly confidence?: number
+  readonly detectorName: string
+  readonly detectorVersion?: string
+}
+
+/** Scan text for supported PII findings. */
+export declare function scan(text: string): Array<Finding>
+
+export interface TextRange {
   readonly start: number
   readonly end: number
 }
-
-/** Scan text for supported PII entities. */
-export declare function scan(text: string): Array<Entity>

bindings/node/src/lib.rs

@@ -4,18 +4,36 @@ use napi::{Error, Status};
 use napi_derive::napi;
 
 #[napi(object, object_from_js = false)]
-pub struct Entity {
-    #[napi(readonly, ts_type = "Label")]
-    pub label: String,
+pub struct TextRange {
+    #[napi(readonly)]
+    pub start: u32,
+
+    #[napi(readonly)]
+    pub end: u32,
+}
+
+#[napi(object, object_from_js = false)]
+pub struct Finding {
+    #[napi(readonly, ts_type = "EntityType")]
+    pub entity_type: String,
 
     #[napi(readonly)]
-    pub text: String,
+    pub matched_text: String,
 
     #[napi(readonly)]
-    pub start: u32,
+    pub byte_range: TextRange,
 
     #[napi(readonly)]
-    pub end: u32,
+    pub codepoint_range: TextRange,
+
+    #[napi(readonly)]
+    pub confidence: Option<f64>,
+
+    #[napi(readonly)]
+    pub detector_name: String,
+
+    #[napi(readonly)]
+    pub detector_version: Option<String>,
 }
 
 fn js_offset(offset: usize) -> napi::Result<u32> {
@@ -27,17 +45,27 @@ fn js_offset(offset: usize) -> napi::Result<u32> {
     })
 }
 
-/// Scan text for supported PII entities.
+fn js_range(range: datafog_core::TextRange) -> napi::Result<TextRange> {
+    Ok(TextRange {
+        start: js_offset(range.start)?,
+        end: js_offset(range.end)?,
+    })
+}
+
+/// Scan text for supported PII findings.
 #[napi(strict, catch_unwind)]
-pub fn scan(text: String) -> napi::Result<Vec<Entity>> {
+pub fn scan(text: String) -> napi::Result<Vec<Finding>> {
     datafog_core::scan(&text)
         .into_iter()
-        .map(|entity| {
-            Ok(Entity {
-                label: entity.label,
-                text: entity.text,
-                start: js_offset(entity.start)?,
-                end: js_offset(entity.end)?,
+        .map(|finding| {
+            Ok(Finding {
+                entity_type: finding.entity_type,
+                matched_text: finding.matched_text,
+                byte_range: js_range(finding.byte_range)?,
+                codepoint_range: js_range(finding.codepoint_range)?,
+                confidence: finding.confidence.map(f64::from),
+                detector_name: finding.detector_name,
+                detector_version: finding.detector_version,
             })
         })
         .collect()

bindings/python/src/lib.rs

@@ -2,61 +2,116 @@ use ::datafog_core as core;
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
 
-/// An entity detected in input text.
+/// A zero-based, end-exclusive text range.
+#[pyclass(frozen, skip_from_py_object)]
+#[derive(Clone, PartialEq, Eq)]
+struct TextRange {
+    #[pyo3(get)]
+    start: usize,
+
+    #[pyo3(get)]
+    end: usize,
+}
+
+impl From<core::TextRange> for TextRange {
+    fn from(range: core::TextRange) -> Self {
+        Self {
+            start: range.start,
+            end: range.end,
+        }
+    }
+}
+
+#[pymethods]
+impl TextRange {
+    fn __repr__(&self) -> String {
+        format!("TextRange(start={}, end={})", self.start, self.end)
+    }
+
+    fn __eq__(&self, other: PyRef<'_, TextRange>) -> bool {
+        self == &*other
+    }
+}
+
+/// A piece of potentially sensitive content detected in input text.
 #[pyclass(frozen, skip_from_py_object)]
 #[derive(Clone)]
-struct Entity {
+struct Finding {
     #[pyo3(get)]
-    label: String,
+    entity_type: String,
 
     #[pyo3(get)]
-    text: String,
+    matched_text: String,
 
     #[pyo3(get)]
-    start: usize,
+    byte_range: TextRange,
 
     #[pyo3(get)]
-    end: usize,
+    codepoint_range: TextRange,
+
+    #[pyo3(get)]
+    confidence: Option<f32>,
+
+    #[pyo3(get)]
+    detector_name: String,
+
+    #[pyo3(get)]
+    detector_version: Option<String>,
 }
 
-impl From<core::Entity> for Entity {
-    fn from(entity: core::Entity) -> Self {
+impl From<core::Finding> for Finding {
+    fn from(finding: core::Finding) -> Self {
         Self {
-            label: entity.label,
-            text: entity.text,
-            start: entity.start,
-            end: entity.end,
+            entity_type: finding.entity_type,
+            matched_text: finding.matched_text,
+            byte_range: finding.byte_range.into(),
+            codepoint_range: finding.codepoint_range.into(),
+            confidence: finding.confidence,
+            detector_name: finding.detector_name,
+            detector_version: finding.detector_version,
         }
     }
 }
 
 #[pymethods]
-impl Entity {
+impl Finding {
     fn __repr__(&self) -> String {
         format!(
-            "Entity(label={:?}, text={:?}, start={}, end={})",
-            self.label, self.text, self.start, self.end
+            "Finding(entity_type={:?}, matched_text={:?}, byte_range={:?}, \
+             codepoint_range={:?}, confidence={:?}, detector_name={:?}, \
+             detector_version={:?})",
+            self.entity_type,
+            self.matched_text,
+            (self.byte_range.start, self.byte_range.end),
+            (self.codepoint_range.start, self.codepoint_range.end),
+            self.confidence,
+            self.detector_name,
+            self.detector_version,
         )
     }
 
-    fn __eq__(&self, other: PyRef<'_, Entity>) -> bool {
-        self.label == other.label
-            && self.text == other.text
-            && self.start == other.start
-            && self.end == other.end
+    fn __eq__(&self, other: PyRef<'_, Finding>) -> bool {
+        self.entity_type == other.entity_type
+            && self.matched_text == other.matched_text
+            && self.byte_range == other.byte_range
+            && self.codepoint_range == other.codepoint_range
+            && self.confidence == other.confidence
+            && self.detector_name == other.detector_name
+            && self.detector_version == other.detector_version
     }
 }
 
-/// Scan text for supported PII entities.
+/// Scan text for supported PII findings.
 #[pyfunction]
-fn scan(text: &str) -> PyResult<Vec<Entity>> {
-    std::panic::catch_unwind(|| core::scan(text).into_iter().map(Entity::from).collect())
+fn scan(text: &str) -> PyResult<Vec<Finding>> {
+    std::panic::catch_unwind(|| core::scan(text).into_iter().map(Finding::from).collect())
         .map_err(|_| PyRuntimeError::new_err("unexpected Rust scan failure"))
 }
 
 #[pymodule]
 fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
-    module.add_class::<Entity>()?;
+    module.add_class::<TextRange>()?;
+    module.add_class::<Finding>()?;
     module.add_function(wrap_pyfunction!(scan, module)?)?;
     Ok(())
 }

bindings/python/tests/test_installed.py

@@ -17,24 +17,49 @@ def expected_entities(record: dict[str, object]) -> list[dict[str, object]]:
 
 def actual_entities(text: str) -> list[dict[str, object]]:
     return [
-        {"label": entity.label, "text": entity.text, "start": entity.start, "end": entity.end}
-        for entity in scan(text)
+        {
+            "label": finding.entity_type,
+            "text": finding.matched_text,
+            "start": finding.codepoint_range.start,
+            "end": finding.codepoint_range.end,
+        }
+        for finding in scan(text)
     ]
 
 
+def verify_contract(text: str) -> None:
+    encoded = text.encode("utf-8")
+    for finding in scan(text):
+        assert (
+            encoded[finding.byte_range.start : finding.byte_range.end].decode("utf-8")
+            == finding.matched_text
+        )
+        assert (
+            text[finding.codepoint_range.start : finding.codepoint_range.end]
+            == finding.matched_text
+        )
+        assert finding.confidence is None
+        assert finding.detector_name.startswith("datafog-core/")
+        assert finding.detector_version
+
+
 def verify_fixture(name: str) -> None:
     path = ROOT / "fixtures" / name
     for line in path.read_text().splitlines():
         record = json.loads(line)
         actual = actual_entities(record["text"])
         expected = expected_entities(record)
         assert actual == expected, record["id"]
+        verify_contract(record["text"])
 
 
 def main() -> None:
     verify_fixture("development.jsonl")
     verify_fixture("final.jsonl")
-    print("Installed datafog_core wheel matches both fixtures.")
+    emoji_finding = scan("👋 jane@example.com")[0]
+    assert (emoji_finding.byte_range.start, emoji_finding.byte_range.end) == (5, 21)
+    assert (emoji_finding.codepoint_range.start, emoji_finding.codepoint_range.end) == (2, 18)
+    print("Installed datafog_core wheel matches fixtures and the Finding contract.")
 
 
 if __name__ == "__main__":

bindings/wasm/index.d.ts

@@ -1,18 +1,20 @@
-export type Label =
-  | "EMAIL"
-  | "PHONE"
-  | "SSN"
-  | "CREDIT_CARD"
-  | "IP_ADDRESS"
-  | "DATE"
-  | "ZIP_CODE";
+/** Canonical built-in values are uppercase, but custom detectors may add values. */
+export type EntityType = string;
 
-export interface Entity {
-  readonly label: Label;
-  readonly text: string;
+export interface TextRange {
   readonly start: number;
   readonly end: number;
 }
 
+export interface Finding {
+  readonly entityType: EntityType;
+  readonly matchedText: string;
+  readonly byteRange: TextRange;
+  readonly codepointRange: TextRange;
+  readonly confidence?: number;
+  readonly detectorName: string;
+  readonly detectorVersion?: string;
+}
+
 export function init(): Promise<void>;
-export function scan(text: string): Entity[];
+export function scan(text: string): Finding[];