From d42a4c6a7db3888d35aaaf1a0bb800f17e5be690 Mon Sep 17 00:00:00 2001 From: Eugene Feinberg Date: Tue, 20 Jan 2026 11:56:13 -0800 Subject: [PATCH 1/5] Add support for git repos with lfs --- bender-lfs-debug-top | 1 + src/sess.rs | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 160000 bender-lfs-debug-top diff --git a/bender-lfs-debug-top b/bender-lfs-debug-top new file mode 160000 index 00000000..8b45ed73 --- /dev/null +++ b/bender-lfs-debug-top @@ -0,0 +1 @@ +Subproject commit 8b45ed73b0bca48fb9bd03a8695e2340d3a050be diff --git a/src/sess.rs b/src/sess.rs index 964c2fd9..49899ead 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -970,7 +970,13 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { if clear == CheckoutState::ToClone { git.clone() .spawn_with(move |c| { - c.arg("clone") + c.arg("-c") + .arg("filter.lfs.smudge=") + .arg("-c") + .arg("filter.lfs.process=") + .arg("-c") + .arg("filter.lfs.required=false") + .arg("clone") .arg(git.path) .arg(path) .arg("--branch") @@ -984,9 +990,24 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { .await?; local_git .clone() - .spawn_with(move |c| c.arg("checkout").arg(tag_name_2).arg("--force")) + .spawn_with(move |c| { + c.arg("-c") + .arg("filter.lfs.smudge=") + .arg("-c") + .arg("filter.lfs.process=") + .arg("-c") + .arg("filter.lfs.required=false") + .arg("checkout") + .arg(tag_name_2) + .arg("--force") + }) .await?; } + local_git + .clone() + .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) + .await?; + local_git.clone().spawn_with(move |c| c.arg("lfs").arg("pull")).await?; local_git .clone() .spawn_with(move |c| { From 205599d56a010a709a5067a957b570a31f810755 Mon Sep 17 00:00:00 2001 From: Eugene Feinberg Date: Tue, 20 Jan 2026 13:04:10 -0800 Subject: [PATCH 2/5] Fix cargo fmt --- src/sess.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sess.rs b/src/sess.rs index 49899ead..9c9dfe61 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1007,7 +1007,10 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { .clone() .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) .await?; - local_git.clone().spawn_with(move |c| c.arg("lfs").arg("pull")).await?; + local_git + .clone() + .spawn_with(move |c| c.arg("lfs").arg("pull")) + .await?; local_git .clone() .spawn_with(move |c| { From 9f288ed65758b9c6db6027eded1cf5cabde85a55 Mon Sep 17 00:00:00 2001 From: Eugene Feinberg Date: Wed, 21 Jan 2026 11:06:37 -0800 Subject: [PATCH 3/5] Handle lack of git-lfs cleanly and update README --- README.md | 8 ++++++++ bender-lfs-debug-top | 1 - src/diagnostic.rs | 7 +++++++ src/git.rs | 39 +++++++++++++++++++++++++++++++++++++++ src/sess.rs | 34 ++++++++++++++++++++++++++-------- 5 files changed, 80 insertions(+), 9 deletions(-) delete mode 160000 bender-lfs-debug-top diff --git a/README.md b/README.md index e25fcf9b..2e1880bf 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,14 @@ All git tags of the form `vX.Y.Z` are considered a version of the package. [Relevant dependency resolution code](https://github.com/pulp-platform/bender/blob/master/src/resolver.rs) +#### Git LFS Support + +Bender detects if a repository requires Git LFS and if the `git-lfs` tool is installed on your system. + +- If the repository uses LFS (detected via `.gitattributes`) and `git-lfs` is installed, Bender will automatically configure LFS and pull the required files. +- If the repository appears to use LFS but `git-lfs` is **not** installed, Bender will print a warning (`W33`) but proceed with the checkout. In this case, you may end up with pointer files instead of the actual large files, which can cause build failures. +- If the repository does not use LFS, Bender skips LFS operations entirely to save time. + #### Target handling Specified dependencies can be filtered, similar to the sources below. For consistency, this filtering does **NOT** apply during an update, i.e., all dependencies will be accounted for in the Bender.lock file. The target filtering only applies for sources and script outputs. This can be used e.g., to include specific IP only for testing. diff --git a/bender-lfs-debug-top b/bender-lfs-debug-top deleted file mode 160000 index 8b45ed73..00000000 --- a/bender-lfs-debug-top +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8b45ed73b0bca48fb9bd03a8695e2340d3a050be diff --git a/src/diagnostic.rs b/src/diagnostic.rs index 0aebfc64..6ab2f9cf 100644 --- a/src/diagnostic.rs +++ b/src/diagnostic.rs @@ -319,6 +319,13 @@ pub enum Warnings { #[error("Path {} for dependency {} does not exist.", fmt_path!(path.display()), fmt_pkg!(pkg))] #[diagnostic(code(W32))] DepPathMissing { pkg: String, path: PathBuf }, + + #[error("Dependency {} seems to use git-lfs, but git-lfs is not installed.", fmt_pkg!(pkg))] + #[diagnostic( + code(W33), + help("Install git-lfs to ensure all files are fetched correctly.") + )] + LfsMissing { pkg: String }, } #[cfg(test)] diff --git a/src/git.rs b/src/git.rs index b11e5af0..f73ed8b6 100644 --- a/src/git.rs +++ b/src/git.rs @@ -157,6 +157,45 @@ impl<'ctx> Git<'ctx> { Ok(()) } + /// Check if git-lfs is available. + pub async fn has_lfs(self) -> bool { + self.spawn_with(|c| c.arg("lfs").arg("version")) + .await + .is_ok() + } + + /// Check if the repository uses LFS. + pub async fn uses_lfs(self) -> Result { + let output = self.spawn_with(|c| c.arg("lfs").arg("ls-files")).await?; + Ok(!output.trim().is_empty()) + } + + /// Check if the repository has LFS attributes configured. + pub async fn uses_lfs_attributes(self) -> Result { + // We use tokio::task::spawn_blocking because walkdir is synchronous + // and file I/O should not block the async runtime. + let path = self.path.to_path_buf(); + tokio::task::spawn_blocking(move || { + use walkdir::WalkDir; + for entry in WalkDir::new(&path) { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + if entry.file_type().is_file() && entry.file_name() == ".gitattributes" { + if let Ok(content) = std::fs::read_to_string(entry.path()) { + if content.contains("filter=lfs") { + return Ok(true); + } + } + } + } + Ok(false) + }) + .await + .map_err(|e| Error::chain("Failed to join blocking task", e))? + } + /// Fetch the tags and refs of a remote. pub async fn fetch(self, remote: &str) -> Result<()> { let r1 = String::from(remote); diff --git a/src/sess.rs b/src/sess.rs index 9c9dfe61..f65a1835 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1003,14 +1003,32 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { }) .await?; } - local_git - .clone() - .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) - .await?; - local_git - .clone() - .spawn_with(move |c| c.arg("lfs").arg("pull")) - .await?; + if local_git.clone().has_lfs().await { + // Check if the repo actually tracks files with LFS + let uses_lfs = local_git.clone().uses_lfs().await.unwrap_or(false); + if uses_lfs { + local_git + .clone() + .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) + .await?; + local_git + .clone() + .spawn_with(move |c| c.arg("lfs").arg("pull")) + .await?; + } + } else { + if local_git + .clone() + .uses_lfs_attributes() + .await + .unwrap_or(false) + { + Warnings::LfsMissing { + pkg: name.to_string(), + } + .emit(); + } + } local_git .clone() .spawn_with(move |c| { From a12d5e0338d6c4c979ef30d551d3e9f11d33a658 Mon Sep 17 00:00:00 2001 From: Michael Rogenmoser Date: Thu, 22 Jan 2026 12:18:45 +0100 Subject: [PATCH 4/5] Adjust lfs checking --- src/diagnostic.rs | 16 ++++++++-------- src/git.rs | 28 +++++++++------------------- src/sess.rs | 39 +++++++++++++++++---------------------- 3 files changed, 34 insertions(+), 49 deletions(-) diff --git a/src/diagnostic.rs b/src/diagnostic.rs index 6ab2f9cf..9141706c 100644 --- a/src/diagnostic.rs +++ b/src/diagnostic.rs @@ -305,9 +305,16 @@ pub enum Warnings { IncludeDirMissing(PathBuf), #[error("Skipping dirty dependency {}", fmt_pkg!(pkg))] - #[diagnostic(help("Use `--no-skip` to still snapshot {}.", fmt_pkg!(pkg)))] + #[diagnostic(code(W25), help("Use `--no-skip` to still snapshot {}.", fmt_pkg!(pkg)))] SkippingDirtyDep { pkg: String }, + #[error("Dependency {} seems to use git-lfs, but git-lfs failed with `{}`.", fmt_pkg!(.0), .1)] + #[diagnostic( + code(W26), + help("You may need to install git-lfs to ensure all files are fetched correctly.") + )] + LfsMissing(String, String), + #[error("File not added, ignoring: {cause}")] #[diagnostic(code(W30))] IgnoredPath { cause: String }, @@ -319,13 +326,6 @@ pub enum Warnings { #[error("Path {} for dependency {} does not exist.", fmt_path!(path.display()), fmt_pkg!(pkg))] #[diagnostic(code(W32))] DepPathMissing { pkg: String, path: PathBuf }, - - #[error("Dependency {} seems to use git-lfs, but git-lfs is not installed.", fmt_pkg!(pkg))] - #[diagnostic( - code(W33), - help("Install git-lfs to ensure all files are fetched correctly.") - )] - LfsMissing { pkg: String }, } #[cfg(test)] diff --git a/src/git.rs b/src/git.rs index f73ed8b6..96182622 100644 --- a/src/git.rs +++ b/src/git.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use futures::TryFutureExt; use tokio::process::Command; use tokio::sync::Semaphore; +use walkdir::WalkDir; use crate::error::*; @@ -157,13 +158,6 @@ impl<'ctx> Git<'ctx> { Ok(()) } - /// Check if git-lfs is available. - pub async fn has_lfs(self) -> bool { - self.spawn_with(|c| c.arg("lfs").arg("version")) - .await - .is_ok() - } - /// Check if the repository uses LFS. pub async fn uses_lfs(self) -> Result { let output = self.spawn_with(|c| c.arg("lfs").arg("ls-files")).await?; @@ -176,24 +170,20 @@ impl<'ctx> Git<'ctx> { // and file I/O should not block the async runtime. let path = self.path.to_path_buf(); tokio::task::spawn_blocking(move || { - use walkdir::WalkDir; - for entry in WalkDir::new(&path) { - let entry = match entry { - Ok(e) => e, - Err(_) => continue, - }; + Ok(WalkDir::new(&path).into_iter().flatten().any(|entry| { if entry.file_type().is_file() && entry.file_name() == ".gitattributes" { if let Ok(content) = std::fs::read_to_string(entry.path()) { - if content.contains("filter=lfs") { - return Ok(true); - } + content.contains("filter=lfs") + } else { + false } + } else { + false } - } - Ok(false) + })) }) .await - .map_err(|e| Error::chain("Failed to join blocking task", e))? + .map_err(|cause| Error::chain("Failed to join blocking task", cause))? } /// Fetch the tags and refs of a remote. diff --git a/src/sess.rs b/src/sess.rs index f65a1835..aa9d97f8 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1003,30 +1003,25 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { }) .await?; } - if local_git.clone().has_lfs().await { + // Check if the repo uses LFS attributes + if local_git.clone().uses_lfs_attributes().await? { // Check if the repo actually tracks files with LFS - let uses_lfs = local_git.clone().uses_lfs().await.unwrap_or(false); - if uses_lfs { - local_git - .clone() - .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) - .await?; - local_git - .clone() - .spawn_with(move |c| c.arg("lfs").arg("pull")) - .await?; - } - } else { - if local_git - .clone() - .uses_lfs_attributes() - .await - .unwrap_or(false) - { - Warnings::LfsMissing { - pkg: name.to_string(), + let uses_lfs = local_git.clone().uses_lfs().await; + match uses_lfs { + Ok(true) => { + local_git + .clone() + .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) + .await?; + local_git + .clone() + .spawn_with(move |c| c.arg("lfs").arg("pull")) + .await?; + } + Ok(false) => {} + Err(cause) => { + Warnings::LfsMissing(name.to_string(), cause.to_string()).emit(); } - .emit(); } } local_git From b4d65e8e1667f08c8eec5803f43c49cf5dad5060 Mon Sep 17 00:00:00 2001 From: Michael Rogenmoser Date: Thu, 22 Jan 2026 12:35:49 +0100 Subject: [PATCH 5/5] Add configuration to disable git lfs --- README.md | 14 ++++++++++++++ src/cli.rs | 1 + src/config.rs | 11 +++++++++++ src/diagnostic.rs | 7 +++++++ src/sess.rs | 36 ++++++++++++++++++++---------------- 5 files changed, 53 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2e1880bf..0cf4a9c9 100644 --- a/README.md +++ b/README.md @@ -405,6 +405,20 @@ overrides: # DEPRECATED: This will be removed at some point. plugins: additional-tools: { path: "/usr/local/additional-tools" } + +# Number of parallel git tasks. Optional. +# Default: 4 +# The number of parallel git operations executed by bender can be adjusted to +# manage performance and load on git servers. Can be overriden as a command +# line argument. +git_throttle: 2 + +# Enable git lfs. Optional. +# Default: true +# Some git dependencies may use git-lfs for additional source files. As +# fetching these files may not always be desired or requried, it can be +# disabled. For multiple conflicting settings will use true. +git_lfs: false ``` [Relevant code](https://github.com/pulp-platform/bender/blob/master/src/config.rs) diff --git a/src/cli.rs b/src/cli.rs index cb45ed07..74fdbdeb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -483,6 +483,7 @@ fn load_config(from: &Path, warn_config_loaded: bool) -> Result { overrides: None, plugins: None, git_throttle: None, + git_lfs: None, }; out = out.merge(default_cfg); diff --git a/src/config.rs b/src/config.rs index 184dcd59..a1cfe28e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1439,6 +1439,8 @@ pub struct Config { pub plugins: IndexMap, /// The git throttle value to use unless overridden by the user. pub git_throttle: Option, + /// Enable git LFS support, requires git-lfs (default: true) + pub git_lfs: bool, } /// A partial configuration. @@ -1454,6 +1456,8 @@ pub struct PartialConfig { pub plugins: Option>, /// The git throttle value to use unless overridden by the user. pub git_throttle: Option, + /// Enable git LFS support, requires git-lfs (default: true) + pub git_lfs: Option, } impl PartialConfig { @@ -1465,6 +1469,7 @@ impl PartialConfig { overrides: None, plugins: None, git_throttle: None, + git_lfs: None, } } } @@ -1508,6 +1513,11 @@ impl Merge for PartialConfig { (None, None) => None, }, git_throttle: self.git_throttle.or(other.git_throttle), + git_lfs: match (self.git_lfs, other.git_lfs) { + (Some(v), None) | (None, Some(v)) => Some(v), + (Some(v1), Some(v2)) => Some(v1 | v2), + (None, None) => None, + }, } } } @@ -1540,6 +1550,7 @@ impl Validate for PartialConfig { None => IndexMap::new(), }, git_throttle: self.git_throttle, + git_lfs: self.git_lfs.unwrap_or(true), }) } } diff --git a/src/diagnostic.rs b/src/diagnostic.rs index 9141706c..c88cfa10 100644 --- a/src/diagnostic.rs +++ b/src/diagnostic.rs @@ -315,6 +315,13 @@ pub enum Warnings { )] LfsMissing(String, String), + #[error("Git LFS is disabled but dependency {} seems to use git-lfs.", fmt_pkg!(.0))] + #[diagnostic( + code(W27), + help("Enable git-lfs support in the configuration to fetch all files correctly.") + )] + LfsDisabled(String), + #[error("File not added, ignoring: {cause}")] #[diagnostic(code(W30))] IgnoredPath { cause: String }, diff --git a/src/sess.rs b/src/sess.rs index aa9d97f8..3d7b3567 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1005,23 +1005,27 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { } // Check if the repo uses LFS attributes if local_git.clone().uses_lfs_attributes().await? { - // Check if the repo actually tracks files with LFS - let uses_lfs = local_git.clone().uses_lfs().await; - match uses_lfs { - Ok(true) => { - local_git - .clone() - .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) - .await?; - local_git - .clone() - .spawn_with(move |c| c.arg("lfs").arg("pull")) - .await?; - } - Ok(false) => {} - Err(cause) => { - Warnings::LfsMissing(name.to_string(), cause.to_string()).emit(); + if self.sess.config.git_lfs { + // Check if the repo actually tracks files with LFS + let uses_lfs = local_git.clone().uses_lfs().await; + match uses_lfs { + Ok(true) => { + local_git + .clone() + .spawn_with(move |c| c.arg("config").arg("lfs.url").arg(url)) + .await?; + local_git + .clone() + .spawn_with(move |c| c.arg("lfs").arg("pull")) + .await?; + } + Ok(false) => {} + Err(cause) => { + Warnings::LfsMissing(name.to_string(), cause.to_string()).emit(); + } } + } else { + Warnings::LfsDisabled(name.to_string()).emit(); } } local_git