From 7a53515bd81664b6dfea0158659a978b041cafa1 Mon Sep 17 00:00:00 2001 From: dario curreri Date: Wed, 22 Jul 2026 12:40:00 +0200 Subject: [PATCH 1/2] feat: allow configuring datafusion-cli history file location datafusion-cli hardcoded the interactive shell's rustyline history as the relative path `.history`, so every launch directory silently got its own history file with no way to point it elsewhere. Add a `--history-file ` flag and a `DATAFUSION_HISTORY_FILE` environment variable (flag takes precedence), keeping `.history` as the default so existing behavior is unchanged. Closes #23796. --- .../examples/cli-session-context.rs | 4 ++- datafusion-cli/src/exec.rs | 7 ++-- datafusion-cli/src/main.rs | 35 +++++++++++++++++-- docs/source/user-guide/cli/usage.md | 2 ++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/datafusion-cli/examples/cli-session-context.rs b/datafusion-cli/examples/cli-session-context.rs index 6095072163870..fbc82923726e2 100644 --- a/datafusion-cli/examples/cli-session-context.rs +++ b/datafusion-cli/examples/cli-session-context.rs @@ -94,5 +94,7 @@ pub async fn main() { instrumented_registry: Arc::new(InstrumentedObjectStoreRegistry::new()), }; - exec_from_repl(&my_ctx, &mut print_options).await.unwrap(); + exec_from_repl(&my_ctx, &mut print_options, None) + .await + .unwrap(); } diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index bc2c15f48debb..42f7dcb88f044 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -48,6 +48,7 @@ use std::collections::HashMap; use std::fs::File; use std::io::BufReader; use std::io::prelude::*; +use std::path::Path; use tokio::signal; /// run and execute SQL statements and commands, against a context with the given print options @@ -129,13 +130,15 @@ pub async fn exec_from_files( pub async fn exec_from_repl( ctx: &dyn CliSessionContext, print_options: &mut PrintOptions, + history_file: Option<&Path>, ) -> rustyline::Result<()> { + let history_file = history_file.unwrap_or_else(|| Path::new(".history")); let mut rl = Editor::new()?; rl.set_helper(Some(CliHelper::new( &ctx.task_ctx().session_config().options().sql_parser.dialect, print_options.color, ))); - rl.load_history(".history").ok(); + rl.load_history(history_file).ok(); loop { match rl.readline("> ") { @@ -210,7 +213,7 @@ pub async fn exec_from_repl( } } - rl.save_history(".history") + rl.save_history(history_file) } pub(super) async fn exec_and_print( diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 20a2537d7c10c..00739e2951453 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use std::env; use std::num::NonZeroUsize; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::sync::{Arc, LazyLock}; @@ -109,6 +109,13 @@ struct Args { )] rc: Option>, + #[clap( + long, + help = "Path to the file used to persist interactive shell history. Overrides DATAFUSION_HISTORY_FILE if set, default to .history", + value_parser(parse_valid_history_file) + )] + history_file: Option, + #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] format: PrintFormat, @@ -304,12 +311,17 @@ async fn main_inner() -> Result<()> { } }; + let history_file = args + .history_file + .map(PathBuf::from) + .or_else(|| env::var_os("DATAFUSION_HISTORY_FILE").map(PathBuf::from)); + if repl_mode { if !rc.is_empty() { exec::exec_from_files(&ctx, rc, &print_options).await?; } // TODO maybe we can have thiserror for cli but for now let's keep it simple - return exec::exec_from_repl(&ctx, &mut print_options) + return exec::exec_from_repl(&ctx, &mut print_options, history_file.as_deref()) .await .map_err(|e| DataFusionError::External(Box::new(e))); } @@ -379,6 +391,25 @@ fn parse_valid_data_dir(dir: &str) -> Result { } } +fn parse_valid_history_file(path: &str) -> Result { + let path = Path::new(path); + if path.is_dir() { + return Err(format!( + "Invalid history file '{}': is a directory", + path.display() + )); + } + match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() && !parent.is_dir() => { + Err(format!( + "Invalid history file '{}': parent directory does not exist", + path.display() + )) + } + _ => Ok(path.to_string_lossy().into_owned()), + } +} + fn parse_batch_size(size: &str) -> Result { match size.parse::() { Ok(size) if size > 0 => Ok(size), diff --git a/docs/source/user-guide/cli/usage.md b/docs/source/user-guide/cli/usage.md index 75c6698f007a5..a4fe62f17f29b 100644 --- a/docs/source/user-guide/cli/usage.md +++ b/docs/source/user-guide/cli/usage.md @@ -39,6 +39,8 @@ Options: Execute commands from file(s), then exit -r, --rc [...] Run the provided files on startup instead of ~/.datafusionrc + --history-file + Path to the file used to persist interactive shell history. Overrides DATAFUSION_HISTORY_FILE if set, default to .history --format [default: automatic] [possible values: csv, tsv, table, json, nd-json, automatic] -q, --quiet From 55c1045b039a838405c341cae7517c1825b848e0 Mon Sep 17 00:00:00 2001 From: dario curreri Date: Sat, 25 Jul 2026 12:42:19 +0200 Subject: [PATCH 2/2] feat: introduce ReplOptions for configurable history file in CLI This update introduces a new `ReplOptions` struct to manage the history file configuration for the DataFusion CLI. The `exec_from_repl` function has been modified to accept `ReplOptions`, allowing users to specify a custom history file path. The default behavior remains unchanged, defaulting to `.history` if no path is provided. This change enhances the flexibility of the CLI's interactive shell experience. --- .../examples/cli-session-context.rs | 3 +- datafusion-cli/src/exec.rs | 9 +++-- datafusion-cli/src/lib.rs | 1 + datafusion-cli/src/main.rs | 20 +++++++---- datafusion-cli/src/repl_options.rs | 33 +++++++++++++++++++ 5 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 datafusion-cli/src/repl_options.rs diff --git a/datafusion-cli/examples/cli-session-context.rs b/datafusion-cli/examples/cli-session-context.rs index fbc82923726e2..0300e9118f54a 100644 --- a/datafusion-cli/examples/cli-session-context.rs +++ b/datafusion-cli/examples/cli-session-context.rs @@ -94,7 +94,8 @@ pub async fn main() { instrumented_registry: Arc::new(InstrumentedObjectStoreRegistry::new()), }; - exec_from_repl(&my_ctx, &mut print_options, None) + let repl_options = Default::default(); + exec_from_repl(&my_ctx, &mut print_options, &repl_options) .await .unwrap(); } diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index 42f7dcb88f044..3846341c429c9 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -20,6 +20,7 @@ use crate::cli_context::CliSessionContext; use crate::helper::split_from_semicolon; use crate::print_format::PrintFormat; +use crate::repl_options::ReplOptions; use crate::{ command::{Command, OutputFormat}, helper::CliHelper, @@ -48,7 +49,6 @@ use std::collections::HashMap; use std::fs::File; use std::io::BufReader; use std::io::prelude::*; -use std::path::Path; use tokio::signal; /// run and execute SQL statements and commands, against a context with the given print options @@ -130,15 +130,14 @@ pub async fn exec_from_files( pub async fn exec_from_repl( ctx: &dyn CliSessionContext, print_options: &mut PrintOptions, - history_file: Option<&Path>, + repl_options: &ReplOptions, ) -> rustyline::Result<()> { - let history_file = history_file.unwrap_or_else(|| Path::new(".history")); let mut rl = Editor::new()?; rl.set_helper(Some(CliHelper::new( &ctx.task_ctx().session_config().options().sql_parser.dialect, print_options.color, ))); - rl.load_history(history_file).ok(); + rl.load_history(&repl_options.history_file).ok(); loop { match rl.readline("> ") { @@ -213,7 +212,7 @@ pub async fn exec_from_repl( } } - rl.save_history(history_file) + rl.save_history(&repl_options.history_file) } pub(super) async fn exec_and_print( diff --git a/datafusion-cli/src/lib.rs b/datafusion-cli/src/lib.rs index f0b0bc23fd73d..5182a7600ae79 100644 --- a/datafusion-cli/src/lib.rs +++ b/datafusion-cli/src/lib.rs @@ -34,3 +34,4 @@ pub mod object_storage; pub mod pool_type; pub mod print_format; pub mod print_options; +pub mod repl_options; diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 00739e2951453..417cb9ed14b45 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -43,6 +43,7 @@ use datafusion_cli::{ pool_type::PoolType, print_format::PrintFormat, print_options::{MaxRows, PrintOptions}, + repl_options::ReplOptions, }; use clap::Parser; @@ -311,17 +312,14 @@ async fn main_inner() -> Result<()> { } }; - let history_file = args - .history_file - .map(PathBuf::from) - .or_else(|| env::var_os("DATAFUSION_HISTORY_FILE").map(PathBuf::from)); - if repl_mode { if !rc.is_empty() { exec::exec_from_files(&ctx, rc, &print_options).await?; } + let repl_options = get_repl_options(&args.history_file); + // TODO maybe we can have thiserror for cli but for now let's keep it simple - return exec::exec_from_repl(&ctx, &mut print_options, history_file.as_deref()) + return exec::exec_from_repl(&ctx, &mut print_options, &repl_options) .await .map_err(|e| DataFusionError::External(Box::new(e))); } @@ -425,6 +423,16 @@ fn parse_command(command: &str) -> Result { } } +fn get_repl_options(history_file: &Option) -> ReplOptions { + if let Some(history_file) = history_file { + ReplOptions { + history_file: PathBuf::from(history_file), + } + } else { + Default::default() + } +} + #[derive(Debug, Clone, Copy)] enum ByteUnit { Byte, diff --git a/datafusion-cli/src/repl_options.rs b/datafusion-cli/src/repl_options.rs new file mode 100644 index 0000000000000..beef2a9820462 --- /dev/null +++ b/datafusion-cli/src/repl_options.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::env; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub struct ReplOptions { + pub history_file: PathBuf, +} + +impl Default for ReplOptions { + fn default() -> Self { + let history_file = env::var_os("DATAFUSION_HISTORY_FILE") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".history")); + Self { history_file } + } +}