feat: make object_store support optional

.github/workflows/rust.yml

@@ -222,6 +222,19 @@ jobs:
       #
       - name: Check datafusion (no-default-features)
         run: cargo xtask ci step check datafusion no-default
+      - name: Check datafusion (object_store)
+        run: cargo xtask ci step check datafusion object_store
+      - name: Check datafusion does not require object_store
+        run: |
+          tree_file=$(mktemp)
+          trap 'rm -f "$tree_file"' EXIT
+          cargo tree -p datafusion --no-default-features --features sql \
+            --target all --edges all --prefix none --format '{p}' > "$tree_file"
+          if grep -q '^object_store v' "$tree_file"; then
+            cat "$tree_file"
+            echo "DataFusion without storage must not depend on object_store" >&2
+            exit 1
+          fi
       - name: Check datafusion (nested_expressions)
         run: cargo xtask ci step check datafusion nested_expressions
       - name: Check datafusion (array_expressions)

Cargo.toml

@@ -123,7 +123,7 @@ criterion = "0.8"
 ctor = "1.0.7"
 dashmap = "6.2.1"
 datafusion = { path = "datafusion/core", version = "55.0.0", default-features = false }
-datafusion-catalog = { path = "datafusion/catalog", version = "55.0.0" }
+datafusion-catalog = { path = "datafusion/catalog", version = "55.0.0", default-features = false }
 datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "55.0.0" }
 datafusion-common = { path = "datafusion/common", version = "55.0.0", default-features = false }
 datafusion-common-runtime = { path = "datafusion/common-runtime", version = "55.0.0" }
@@ -138,27 +138,38 @@ datafusion-execution = { path = "datafusion/execution", version = "55.0.0", defa
 datafusion-expr = { path = "datafusion/expr", version = "55.0.0", default-features = false }
 datafusion-expr-common = { path = "datafusion/expr-common", version = "55.0.0" }
 datafusion-ffi = { path = "datafusion/ffi", version = "55.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "55.0.0" }
-datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "55.0.0" }
+# Select the existing expression defaults without implicitly enabling storage.
+datafusion-functions = { path = "datafusion/functions", version = "55.0.0", default-features = false, features = [
+    "datetime_expressions",
+    "encoding_expressions",
+    "math_expressions",
+    "regex_expressions",
+    "string_expressions",
+    "unicode_expressions",
+] }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "55.0.0", default-features = false }
 datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "55.0.0" }
 datafusion-functions-nested = { path = "datafusion/functions-nested", version = "55.0.0", default-features = false }
-datafusion-functions-table = { path = "datafusion/functions-table", version = "55.0.0" }
+datafusion-functions-table = { path = "datafusion/functions-table", version = "55.0.0", default-features = false }
 datafusion-functions-window = { path = "datafusion/functions-window", version = "55.0.0" }
 datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "55.0.0" }
 datafusion-macros = { path = "datafusion/macros", version = "55.0.0" }
 datafusion-optimizer = { path = "datafusion/optimizer", version = "55.0.0", default-features = false }
 datafusion-physical-expr = { path = "datafusion/physical-expr", version = "55.0.0", default-features = false }
 datafusion-physical-expr-adapter = { path = "datafusion/physical-expr-adapter", version = "55.0.0", default-features = false }
 datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "55.0.0", default-features = false }
-datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "55.0.0" }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "55.0.0" }
+datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "55.0.0", default-features = false }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "55.0.0", default-features = false }
 datafusion-proto = { path = "datafusion/proto", version = "55.0.0", default-features = false }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "55.0.0" }
 datafusion-proto-models = { path = "datafusion/proto-models", version = "55.0.0" }
-datafusion-pruning = { path = "datafusion/pruning", version = "55.0.0" }
-datafusion-session = { path = "datafusion/session", version = "55.0.0" }
+datafusion-pruning = { path = "datafusion/pruning", version = "55.0.0", default-features = false }
+datafusion-session = { path = "datafusion/session", version = "55.0.0", default-features = false }
 datafusion-spark = { path = "datafusion/spark", version = "55.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "55.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "55.0.0", default-features = false, features = [
+    "unicode_expressions",
+    "unparser",
+] }
 datafusion-substrait = { path = "datafusion/substrait", version = "55.0.0" }
 
 doc-comment = "0.3"

datafusion/catalog-listing/Cargo.toml

@@ -33,10 +33,10 @@ all-features = true
 [dependencies]
 arrow = { workspace = true }
 async-trait = { workspace = true }
-datafusion-catalog = { workspace = true }
+datafusion-catalog = { workspace = true, features = ["object_store"] }
 datafusion-common = { workspace = true, features = ["object_store"] }
-datafusion-datasource = { workspace = true }
-datafusion-execution = { workspace = true }
+datafusion-datasource = { workspace = true, features = ["object_store"] }
+datafusion-execution = { workspace = true, features = ["object_store"] }
 datafusion-expr = { workspace = true }
 datafusion-physical-expr = { workspace = true }
 datafusion-physical-expr-adapter = { workspace = true }

datafusion/catalog/Cargo.toml

@@ -30,6 +30,16 @@ version.workspace = true
 [package.metadata.docs.rs]
 all-features = true
 
+[features]
+default = ["object_store"]
+object_store = [
+    "dep:object_store",
+    "datafusion-datasource/object_store",
+    "datafusion-execution/object_store",
+    "datafusion-physical-plan/object_store",
+    "datafusion-session/object_store",
+]
+
 [dependencies]
 arrow = { workspace = true }
 async-trait = { workspace = true }
@@ -45,7 +55,7 @@ datafusion-session = { workspace = true }
 futures = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
-object_store = { workspace = true }
+object_store = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 tokio = { workspace = true }
 

datafusion/catalog/src/lib.rs

@@ -39,6 +39,7 @@ pub mod cte_worktable;
 pub mod default_table_source;
 pub mod empty;
 pub mod information_schema;
+#[cfg(feature = "object_store")]
 pub mod listing_schema;
 pub mod memory;
 pub mod stream;

datafusion/core/Cargo.toml

@@ -50,19 +50,42 @@ nested_expressions = ["datafusion-functions-nested"]
 # This feature is deprecated. Use the `nested_expressions` feature instead.
 array_expressions = ["nested_expressions"]
 # Used to enable the avro format
-avro = ["datafusion-datasource-avro"]
+avro = ["object_store", "datafusion-datasource-avro"]
 backtrace = ["datafusion-common/backtrace"]
 compression = [
     "liblzma",
     "bzip2",
     "flate2",
     "zstd",
-    "datafusion-datasource-arrow/compression",
+    "datafusion-datasource-arrow?/compression",
     "datafusion-datasource/compression",
 ]
 crypto_expressions = ["datafusion-functions/crypto_expressions"]
 datetime_expressions = ["datafusion-functions/datetime_expressions"]
+# Built-in file sources, object store registration, and file caches.
+# Disable default features and omit this feature for storage-independent embedding.
+object_store = [
+    "dep:object_store",
+    "datafusion-common/object_store",
+    "datafusion-execution/object_store",
+    "datafusion-datasource/object_store",
+    "datafusion-catalog/object_store",
+    "dep:datafusion-catalog-listing",
+    "dep:datafusion-datasource-arrow",
+    "dep:datafusion-datasource-csv",
+    "dep:datafusion-datasource-json",
+    "datafusion-functions/object_store",
+    "datafusion-functions-aggregate/object_store",
+    "datafusion-functions-nested?/object_store",
+    "datafusion-functions-table/object_store",
+    "datafusion-physical-expr-adapter/object_store",
+    "datafusion-physical-optimizer/object_store",
+    "datafusion-physical-plan/object_store",
+    "datafusion-session/object_store",
+    "datafusion-sql?/object_store",
+]
 default = [
+    "object_store",
     "nested_expressions",
     "crypto_expressions",
     "datetime_expressions",
@@ -79,7 +102,7 @@ encoding_expressions = ["datafusion-functions/encoding_expressions"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = ["datafusion-physical-plan/force_hash_collisions", "datafusion-common/force_hash_collisions"]
 math_expressions = ["datafusion-functions/math_expressions"]
-parquet = ["datafusion-common/parquet", "dep:parquet", "datafusion-datasource-parquet"]
+parquet = ["object_store", "datafusion-common/parquet", "dep:parquet", "datafusion-datasource-parquet"]
 parquet_encryption = [
     "parquet",
     "parquet/encryption",
@@ -124,14 +147,14 @@ async-trait = { workspace = true }
 bzip2 = { workspace = true, optional = true }
 chrono = { workspace = true }
 datafusion-catalog = { workspace = true }
-datafusion-catalog-listing = { workspace = true }
-datafusion-common = { workspace = true, features = ["object_store"] }
+datafusion-catalog-listing = { workspace = true, optional = true }
+datafusion-common = { workspace = true }
 datafusion-common-runtime = { workspace = true }
 datafusion-datasource = { workspace = true }
-datafusion-datasource-arrow = { workspace = true }
+datafusion-datasource-arrow = { workspace = true, optional = true }
 datafusion-datasource-avro = { workspace = true, optional = true }
-datafusion-datasource-csv = { workspace = true }
-datafusion-datasource-json = { workspace = true }
+datafusion-datasource-csv = { workspace = true, optional = true }
+datafusion-datasource-json = { workspace = true, optional = true }
 datafusion-datasource-parquet = { workspace = true, optional = true }
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true, default-features = false }
@@ -155,7 +178,7 @@ indexmap = { workspace = true }
 itertools = { workspace = true }
 liblzma = { workspace = true, optional = true }
 log = { workspace = true }
-object_store = { workspace = true }
+object_store = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 parquet = { workspace = true, optional = true, default-features = true }
 serde = { version = "1.0", default-features = false, features = ["derive"], optional = true }

datafusion/core/src/dataframe/mod.rs

@@ -22,8 +22,11 @@ mod parquet;
 
 use crate::arrow::record_batch::RecordBatch;
 use crate::arrow::util::pretty;
+#[cfg(feature = "object_store")]
 use crate::datasource::file_format::csv::CsvFormatFactory;
+#[cfg(feature = "object_store")]
 use crate::datasource::file_format::format_as_file_type;
+#[cfg(feature = "object_store")]
 use crate::datasource::file_format::json::JsonFormatFactory;
 use crate::datasource::{
     DefaultTableSource, MemTable, TableProvider, provider_as_source,
@@ -42,18 +45,23 @@ use crate::physical_plan::{
 };
 use crate::prelude::SessionContext;
 use std::borrow::Cow;
-use std::collections::{HashMap, HashSet};
+#[cfg(feature = "object_store")]
+use std::collections::HashMap;
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use arrow::array::{Array, ArrayRef, Int64Array, StringArray};
 use arrow::compute::{cast, concat};
 use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
 use arrow::util::display::{ArrayFormatter, FormatOptions};
 use arrow_schema::FieldRef;
+#[cfg(feature = "object_store")]
 use datafusion_common::config::{CsvOptions, JsonOptions};
+#[cfg(feature = "object_store")]
+use datafusion_common::not_impl_err;
 use datafusion_common::{
     Column, DFSchema, DataFusionError, ParamValues, ScalarValue, SchemaError,
-    TableReference, UnnestOptions, exec_err, internal_datafusion_err, not_impl_err,
+    TableReference, UnnestOptions, exec_err, internal_datafusion_err,
     plan_datafusion_err, plan_err, unqualified_field_not_found,
 };
 use datafusion_expr::select_expr::SelectExpr;
@@ -133,6 +141,7 @@ impl DataFrameWriteOptions {
     }
 
     /// Build the options HashMap to pass to CopyTo for sink configuration.
+    #[cfg(feature = "object_store")]
     fn build_sink_options(&self) -> HashMap<String, String> {
         let mut options = HashMap::new();
         if let Some(single_file) = self.single_file_output {
@@ -2060,6 +2069,7 @@ impl DataFrame {
     /// # Ok(())
     /// # }
     /// ```
+    #[cfg(feature = "object_store")]
     pub async fn write_csv(
         self,
         path: &str,
@@ -2130,6 +2140,7 @@ impl DataFrame {
     /// # Ok(())
     /// # }
     /// ```
+    #[cfg(feature = "object_store")]
     pub async fn write_json(
         self,
         path: &str,

datafusion/core/src/datasource/mod.rs

@@ -19,11 +19,16 @@
 //!
 //! [`ListingTable`]: crate::datasource::listing::ListingTable
 
+#[cfg(feature = "object_store")]
 pub mod dynamic_file;
+#[cfg(feature = "object_store")]
 pub mod file_format;
+#[cfg(feature = "object_store")]
 pub mod listing;
+#[cfg(feature = "object_store")]
 pub mod listing_table_factory;
 mod memory_test;
+#[cfg(feature = "object_store")]
 pub mod physical_plan;
 pub mod provider;
 mod view_test;
@@ -42,11 +47,13 @@ pub use datafusion_catalog::empty;
 pub use datafusion_catalog::memory;
 pub use datafusion_catalog::stream;
 pub use datafusion_catalog::view;
+#[cfg(feature = "object_store")]
 pub use datafusion_datasource::projection;
 pub use datafusion_datasource::schema_adapter;
 pub use datafusion_datasource::sink;
 pub use datafusion_datasource::source;
 pub use datafusion_datasource::table_schema;
+#[cfg(feature = "object_store")]
 pub use datafusion_execution::object_store;
 pub use datafusion_physical_expr::create_ordering;
 

datafusion/core/src/datasource/provider.rs

@@ -17,36 +17,48 @@
 
 //! Data source traits
 
+#[cfg(feature = "object_store")]
 use std::sync::Arc;
 
+#[cfg(feature = "object_store")]
 use async_trait::async_trait;
+#[cfg(feature = "object_store")]
 use datafusion_catalog::Session;
+#[cfg(feature = "object_store")]
 use datafusion_expr::CreateExternalTable;
 pub use datafusion_expr::{TableProviderFilterPushDown, TableType};
+#[cfg(feature = "object_store")]
 use futures::future::BoxFuture;
 
+#[cfg(feature = "object_store")]
 use crate::catalog::{TableProvider, TableProviderFactory};
+#[cfg(feature = "object_store")]
 use crate::datasource::listing_table_factory::ListingTableFactory;
+#[cfg(feature = "object_store")]
 use crate::datasource::stream::StreamTableFactory;
+#[cfg(feature = "object_store")]
 use crate::error::Result;
 
 /// The default [`TableProviderFactory`]
 ///
 /// If [`CreateExternalTable`] is unbounded calls [`StreamTableFactory::create`],
 /// otherwise calls [`ListingTableFactory::create`]
+#[cfg(feature = "object_store")]
 #[derive(Debug, Default)]
 pub struct DefaultTableFactory {
     stream: StreamTableFactory,
     listing: ListingTableFactory,
 }
 
+#[cfg(feature = "object_store")]
 impl DefaultTableFactory {
     /// Creates a new [`DefaultTableFactory`]
     pub fn new() -> Self {
         Self::default()
     }
 }
 
+#[cfg(feature = "object_store")]
 #[async_trait]
 impl TableProviderFactory for DefaultTableFactory {
     // Hand-written `#[async_trait]` expansion to reduce compile time. See
@@ -66,6 +78,7 @@ impl TableProviderFactory for DefaultTableFactory {
     }
 }
 
+#[cfg(feature = "object_store")]
 impl DefaultTableFactory {
     fn create_boxed<'a>(
         &'a self,