refactor: split a deeptools-core crate with a curated public API

.github/workflows/test_rust.yml

@@ -18,7 +18,15 @@ jobs:
               with:
                 persist-credentials: false
             - run: rustup update stable && rustup default stable
-            - name: cargo build
+            - name: cargo build (core only)
               run: cargo build --verbose
-            - name: cargo test
+            - name: cargo test (core only)
               run: cargo test --verbose
+            - name: cargo build (python extension)
+              run: cargo build --verbose --features python
+            - name: cargo test (python extension)
+              run: cargo test --verbose --features python
+            - name: core crate has no pyo3 dependency
+              run: |
+                cargo tree -e normal --prefix none > tree.txt
+                ! grep -q '^pyo3' tree.txt

Cargo.toml

@@ -1,23 +1,36 @@
 [package]
-name = "hp"
-version = "0.1.0"
+name = "deeptools-core"
+version = "4.0.0"
 edition = "2024"
 rust-version = "1.87"
+description = "Pure-Rust core of deepTools: coverage normalization scale factors"
+license-file = "LICENSE.txt"
+repository = "https://github.com/deeptools/deepTools"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
-name = "hp"
-crate-type = ["cdylib"]
+name = "deeptools_core"
+crate-type = ["rlib", "cdylib"]
+
+[features]
+python = [
+    "dep:pyo3", "pyo3/extension-module",
+    "dep:openssl-sys", "dep:bigtools", "dep:rust-htslib", "dep:rayon", "dep:tokio",
+    "dep:itertools", "dep:flate2", "dep:tempfile", "dep:ndarray-npy",
+]
+default = []
 
 [dependencies]
-openssl-sys = { version = "0.9.117", features = ["vendored"] }
-pyo3 = { version = ">= 0.28.2", features = ["extension-module"] }
-bigtools = { version = "0.5.8", default-features = false, features = ["read", "write"] }
-rust-htslib = "1.0.1"
-rayon = "1.12.0"
-tokio = "1.53.0"
-itertools = "0.15.0"
-flate2 = "1.1.9"
-tempfile = "3.27.0"
+openssl-sys = { version = "0.9.117", features = ["vendored"], optional = true }
+pyo3 = { version = ">= 0.28.2", optional = true }
+bigtools = { version = "0.5.8", default-features = false, features = ["read", "write"], optional = true }
+rust-htslib = { version = "1.0.1", optional = true }
+rayon = { version = "1.12.0", optional = true }
+tokio = { version = "1.53.0", optional = true }
+itertools = { version = "0.15.0", optional = true }
+flate2 = { version = "1.1.9", optional = true }
+tempfile = { version = "3.27.0", optional = true }
 ndarray = "0.17.2"
-ndarray-npy = "0.10.0"
+ndarray-npy = { version = "0.10.0", optional = true }
+thiserror = "2.0"

pyproject.toml

@@ -9,6 +9,7 @@ python-source = "pydeeptools"
 exclude = ["pydeeptools/deeptools/test/**/*"]
 module-name = "deeptools.hp"
 bindings = "pyo3"
+features = ["python"]
 
 [project]
 name = "deeptools"

release-please-config.json

@@ -11,6 +11,16 @@
         {
           "type": "generic",
           "path": "galaxy/wrapper/deepTools_macros.xml"
+        },
+        {
+          "type": "toml",
+          "path": "Cargo.toml",
+          "jsonpath": "$.package.version"
+        },
+        {
+          "type": "toml",
+          "path": "Cargo.lock",
+          "jsonpath": "$.package[?(@.name.value=='deeptools-core')].version"
         }
       ]
     }

scripts/check_version_sync.py

@@ -99,6 +99,20 @@ def galaxy_tool_version() -> str:
     return match.group(1)
 
 
+def cargo_crate_version() -> str:
+    data = tomllib.loads((REPO_ROOT / "Cargo.toml").read_text())
+    return data["package"]["version"]
+
+
+def cargo_lock_crate_version() -> str:
+    name = tomllib.loads((REPO_ROOT / "Cargo.toml").read_text())["package"]["name"]
+    lock = tomllib.loads((REPO_ROOT / "Cargo.lock").read_text())
+    for package in lock["package"]:
+        if package["name"] == name:
+            return package["version"]
+    sys.exit(f"could not find crate {name!r} in Cargo.lock")
+
+
 def main() -> None:
     cargo_version = cargo_rust_version()
     mismatches: dict[str, tuple[str, str]] = {}
@@ -125,6 +139,13 @@ def main() -> None:
             galaxy_version,
         )
 
+    for label, version in (
+        ("deeptools: Cargo.toml [package] version", cargo_crate_version()),
+        ("deeptools: Cargo.lock crate version", cargo_lock_crate_version()),
+    ):
+        if version != pkg_version:
+            mismatches[label] = (pkg_version, version)
+
     if mismatches:
         print("Version mismatch detected:", file=sys.stderr)
         for name, (expected, actual) in mismatches.items():

src/bamcompare.rs

@@ -206,18 +206,24 @@ pub fn r_bamcompare(
         }
         (given_sf1, given_sf2)
     } else {
+        let to_py = |e: crate::normalization::NormalizationError| {
+            pyo3::exceptions::PyValueError::new_err(e.to_string())
+        };
+        let method: crate::normalization::ScaleFactorsMethod =
+            scalefactorsmethod.parse().map_err(to_py)?;
+        let norm_method: crate::normalization::Normalization = norm.parse().map_err(to_py)?;
+        let stats = |i: usize| crate::normalization::SampleStats {
+            mapped: covcalcs[i].mapped,
+            readlen: covcalcs[i].readlen,
+        };
         let (sf1, sf2) = scale_factor_bamcompare(
-            scalefactorsmethod,
-            covcalcs[0].mapped,
-            covcalcs[1].mapped,
+            method,
+            [stats(0), stats(1)],
             binsize,
             effective_genome_size,
-            norm,
-            covcalcs[0].readlen,
-            covcalcs[1].readlen,
-            covcalcs[0].fraglen,
-            covcalcs[1].fraglen,
-        );
+            norm_method,
+        )
+        .map_err(to_py)?;
         if verbose {
             println!("scale factor1 = {}, scale factor2 = {}", sf1, sf2);
         }

src/bamcoverage.rs

@@ -66,10 +66,12 @@ pub fn r_bamcoverage(
     if mnase && !ispe {
         panic!("Error: MNase-seq requires paired-end data.");
     }
-    if norm == "RPGC" && effectivegenomesize == 0 {
-        panic!("Error: Effective genome size is required for RPGC normalization.");
-    }
-    if norm != "None" && scalefactor != 1.0 {
+    let norm_method: crate::normalization::Normalization = norm
+        .parse()
+        .map_err(|e: crate::normalization::NormalizationError| {
+            pyo3::exceptions::PyValueError::new_err(e.to_string())
+        })?;
+    if norm_method != crate::normalization::Normalization::None && scalefactor != 1.0 {
         println!(
             "Warning: You have set a normalization option ({}), but also a scale factor. Only the scale factor will be used",
             norm
@@ -230,16 +232,19 @@ pub fn r_bamcoverage(
         println!("Readlen: {}, Fraglen: {}", readlen, fraglen);
     }
 
+    let preset = if scalefactor != 1.0 { Some(scalefactor) } else { None };
     let sf = scale_factor(
-        norm,
+        norm_method,
         mapped,
         binsize,
         effectivegenomesize,
         readlen,
-        fraglen,
-        scalefactor,
-        &verbose,
-    );
+        preset,
+    )
+    .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+    if verbose {
+        println!("Scale factor: {}", sf);
+    }
 
     // Create output stream
     let lines = bg.into_iter().flat_map(|bg| {

src/calc.rs

@@ -161,6 +161,15 @@ pub fn calc_ratio(
     }
 }
 
+/// DESeq-style median-of-ratios scale factors.
+///
+/// `array2` has one row per bin and one column per sample (raw counts).
+/// Returns one scale factor per sample column: multiply a sample's counts by
+/// its factor to make samples comparable.
+///
+/// Entries `<= 0` are ignored, and so are bins where any sample is zero
+/// (their log geometric mean is infinite). A sample with no usable bins gets
+/// a factor of `1.0`.
 pub fn deseq_scalefactors(array2: &Array2<f32>) -> Array1<f32> {
     let loggeomeans = array2.mapv(|v| v.ln()).sum_axis(Axis(1)) / array2.shape()[1] as f32;
     let masked_array = array2.mapv(|x| if x <= 0.0 { f32::NAN } else { x });

src/lib.rs

@@ -1,17 +1,47 @@
+//! Core of deepTools: pure-Rust building blocks for coverage normalization.
+//!
+//! The crate has no dependency on `pyo3` unless the `python` feature is
+//! enabled; that feature only exists to build the Python extension.
+//!
+//! # Public API
+//!
+//! * [`normalization`]: scale factors (`RPKM`, `CPM`, `BPM`, `RPGC`, read
+//!   count scaling between two samples).
+//! * [`deseq_scalefactors`]: DESeq-style median-of-ratios scale factors.
+//!
+//! Everything else is an implementation detail of the Python extension and
+//! may change without notice.
+#![warn(missing_docs)]
+
+#[cfg(feature = "python")]
 use pyo3::prelude::*;
-mod alignmentsieve;
-mod bamcompare;
-mod bamcoverage;
+#[cfg_attr(not(feature = "python"), allow(dead_code))]
 mod calc;
-mod computematrix;
+#[cfg(feature = "python")]
 mod covcalc;
+#[cfg(feature = "python")]
 mod filehandler;
+#[cfg(feature = "python")]
 mod filtering;
+pub mod normalization;
+
+pub use calc::deseq_scalefactors;
+
+#[cfg(feature = "python")]
+mod alignmentsieve;
+#[cfg(feature = "python")]
+mod bamcompare;
+#[cfg(feature = "python")]
+mod bamcoverage;
+#[cfg(feature = "python")]
+mod computematrix;
+#[cfg(feature = "python")]
 mod multibamsummary;
-mod normalization;
+
 #[cfg(test)]
 mod tests;
 
+#[cfg(feature = "python")]
 #[pymodule]
 fn hp(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(bamcoverage::r_bamcoverage, m)?)?;