Enhance configuration options in README and CLI

- Added examples for configuring processing limits in README.md.
- Introduced new CLI options for max files, max total size, and max directory depth.
- Updated environment variable support in config.py for various limits.
- Modified ingestion functions to accept new parameters for file processing limits.
- Enhanced limit checks in ingestion logic to utilize new configuration options.

Author: Cheelax

README.md

@@ -135,12 +135,58 @@ By default, the digest is written to a text file (`digest.txt`) in your current
 - Use `--output/-o <filename>` to write to a specific file.
 - Use `--output/-o -` to output directly to `STDOUT` (useful for piping to other tools).
 
+Configure processing limits:
+
+```bash
+# Set higher limits for large repositories
+gitingest https://github.com/torvalds/linux \
+  --max-files 100000 \
+  --max-total-size 2147483648 \
+  --max-directory-depth 25
+
+# Process only Python files up to 1MB each
+gitingest /path/to/project \
+  --include-pattern "*.py" \
+  --max-size 1048576 \
+  --max-files 1000
+```
+
 See more options and usage details with:
 
 ```bash
 gitingest --help
 ```
 
+### 🔧 Configuration via Environment Variables
+
+You can configure various limits and settings using environment variables. All configuration environment variables start with the `GITINGEST_` prefix:
+
+**File Processing Configuration:**
+- `GITINGEST_MAX_FILE_SIZE` - Maximum size of a single file to process (default: 10485760 bytes, 10MB)
+- `GITINGEST_MAX_FILES` - Maximum number of files to process (default: 10000)
+- `GITINGEST_MAX_TOTAL_SIZE_BYTES` - Maximum size of output file (default: 524288000 bytes, 500MB)
+- `GITINGEST_MAX_DIRECTORY_DEPTH` - Maximum depth of directory traversal (default: 20)
+- `GITINGEST_DEFAULT_TIMEOUT` - Default operation timeout in seconds (default: 60)
+- `GITINGEST_OUTPUT_FILE_NAME` - Default output filename (default: "digest.txt")
+- `GITINGEST_TMP_BASE_PATH` - Base path for temporary files (default: system temp directory)
+
+**Server Configuration (for self-hosting):**
+- `GITINGEST_MAX_DISPLAY_SIZE` - Maximum size of content to display in UI (default: 300000 bytes)
+- `GITINGEST_DELETE_REPO_AFTER` - Repository cleanup timeout in seconds (default: 3600, 1 hour)
+- `GITINGEST_MAX_FILE_SIZE_KB` - Maximum file size for UI slider in KB (default: 102400, 100MB)
+- `GITINGEST_MAX_SLIDER_POSITION` - Maximum slider position in UI (default: 500)
+
+**Example usage:**
+
+```bash
+# Configure for large scientific repositories
+export GITINGEST_MAX_FILES=50000
+export GITINGEST_MAX_FILE_SIZE=20971520  # 20MB
+export GITINGEST_MAX_TOTAL_SIZE_BYTES=1073741824  # 1GB
+
+gitingest https://github.com/some/large-repo
+```
+
 ## 🐍 Python package usage
 
 ```python
@@ -169,6 +215,15 @@ summary, tree, content = ingest("https://github.com/username/private-repo")
 
 # Include repository submodules
 summary, tree, content = ingest("https://github.com/username/repo-with-submodules", include_submodules=True)
+
+# Configure limits programmatically
+summary, tree, content = ingest(
+    "https://github.com/username/large-repo",
+    max_file_size=20 * 1024 * 1024,  # 20MB per file
+    max_files=50000,                  # 50k files max
+    max_total_size_bytes=1024**3,     # 1GB total
+    max_directory_depth=30            # 30 levels deep
+)
 ```
 
 By default, this won't write a file but can be enabled with the `output` argument.

src/gitingest/cli.py

@@ -9,13 +9,16 @@
 import click
 from typing_extensions import Unpack
 
-from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_NAME
+from gitingest.config import MAX_FILE_SIZE, MAX_FILES, MAX_TOTAL_SIZE_BYTES, MAX_DIRECTORY_DEPTH, OUTPUT_FILE_NAME
 from gitingest.entrypoint import ingest_async
 
 
 class _CLIArgs(TypedDict):
     source: str
     max_size: int
+    max_files: int
+    max_total_size: int
+    max_directory_depth: int
     exclude_pattern: tuple[str, ...]
     include_pattern: tuple[str, ...]
     branch: str | None
@@ -34,6 +37,24 @@ class _CLIArgs(TypedDict):
     show_default=True,
     help="Maximum file size to process in bytes",
 )
+@click.option(
+    "--max-files",
+    default=MAX_FILES,
+    show_default=True,
+    help="Maximum number of files to process",
+)
+@click.option(
+    "--max-total-size",
+    default=MAX_TOTAL_SIZE_BYTES,
+    show_default=True,
+    help="Maximum total size of all files in bytes",
+)
+@click.option(
+    "--max-directory-depth",
+    default=MAX_DIRECTORY_DEPTH,
+    show_default=True,
+    help="Maximum depth of directory traversal",
+)
 @click.option("--exclude-pattern", "-e", multiple=True, help="Shell-style patterns to exclude.")
 @click.option(
     "--include-pattern",
@@ -112,6 +133,9 @@ async def _async_main(
     source: str,
     *,
     max_size: int = MAX_FILE_SIZE,
+    max_files: int = MAX_FILES,
+    max_total_size: int = MAX_TOTAL_SIZE_BYTES,
+    max_directory_depth: int = MAX_DIRECTORY_DEPTH,
     exclude_pattern: tuple[str, ...] | None = None,
     include_pattern: tuple[str, ...] | None = None,
     branch: str | None = None,
@@ -170,6 +194,9 @@ async def _async_main(
         summary, _, _ = await ingest_async(
             source,
             max_file_size=max_size,
+            max_files=max_files,
+            max_total_size_bytes=max_total_size,
+            max_directory_depth=max_directory_depth,
             include_patterns=include_patterns,
             exclude_patterns=exclude_patterns,
             branch=branch,

src/gitingest/config.py

@@ -1,14 +1,34 @@
 """Configuration file for the project."""
 
+import os
 import tempfile
 from pathlib import Path
 
-MAX_FILE_SIZE = 10 * 1024 * 1024  # Maximum size of a single file to process (10 MB)
-MAX_DIRECTORY_DEPTH = 20  # Maximum depth of directory traversal
-MAX_FILES = 10_000  # Maximum number of files to process
-MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024  # Maximum size of output file (500 MB)
-DEFAULT_TIMEOUT = 60  # seconds
+# Helper function to get environment variables with type conversion
+def _get_env_var(key: str, default, cast_func=None):
+    """Get environment variable with GITINGEST_ prefix and optional type casting."""
+    env_key = f"GITINGEST_{key}"
+    value = os.environ.get(env_key)
+    
+    if value is None:
+        return default
+    
+    if cast_func:
+        try:
+            return cast_func(value)
+        except (ValueError, TypeError):
+            print(f"Warning: Invalid value for {env_key}: {value}. Using default: {default}")
+            return default
+    
+    return value
 
-OUTPUT_FILE_NAME = "digest.txt"
+# Configuration with environment variable support
+MAX_FILE_SIZE = _get_env_var("MAX_FILE_SIZE", 10 * 1024 * 1024, int)  # Maximum size of a single file to process (10 MB)
+MAX_DIRECTORY_DEPTH = _get_env_var("MAX_DIRECTORY_DEPTH", 20, int)  # Maximum depth of directory traversal
+MAX_FILES = _get_env_var("MAX_FILES", 10_000, int)  # Maximum number of files to process
+MAX_TOTAL_SIZE_BYTES = _get_env_var("MAX_TOTAL_SIZE_BYTES", 500 * 1024 * 1024, int)  # Maximum size of output file (500 MB)
+DEFAULT_TIMEOUT = _get_env_var("DEFAULT_TIMEOUT", 60, int)  # seconds
 
-TMP_BASE_PATH = Path(tempfile.gettempdir()) / "gitingest"
+OUTPUT_FILE_NAME = _get_env_var("OUTPUT_FILE_NAME", "digest.txt")
+
+TMP_BASE_PATH = Path(_get_env_var("TMP_BASE_PATH", tempfile.gettempdir())) / "gitingest"

src/gitingest/entrypoint.py

@@ -22,6 +22,9 @@ async def ingest_async(
     source: str,
     *,
     max_file_size: int = MAX_FILE_SIZE,
+    max_files: int | None = None,
+    max_total_size_bytes: int | None = None,
+    max_directory_depth: int | None = None,
     include_patterns: str | set[str] | None = None,
     exclude_patterns: str | set[str] | None = None,
     branch: str | None = None,
@@ -77,6 +80,9 @@ async def ingest_async(
     query: IngestionQuery = await parse_query(
         source=source,
         max_file_size=max_file_size,
+        max_files=max_files,
+        max_total_size_bytes=max_total_size_bytes,
+        max_directory_depth=max_directory_depth,
         from_web=False,
         include_patterns=include_patterns,
         ignore_patterns=exclude_patterns,
@@ -101,6 +107,9 @@ def ingest(
     source: str,
     *,
     max_file_size: int = MAX_FILE_SIZE,
+    max_files: int | None = None,
+    max_total_size_bytes: int | None = None,
+    max_directory_depth: int | None = None,
     include_patterns: str | set[str] | None = None,
     exclude_patterns: str | set[str] | None = None,
     branch: str | None = None,
@@ -122,6 +131,12 @@ def ingest(
         The source to analyze, which can be a URL (for a Git repository) or a local directory path.
     max_file_size : int
         Maximum allowed file size for file ingestion. Files larger than this size are ignored (default: 10 MB).
+    max_files : int | None
+        Maximum number of files to process. If ``None``, uses the default from config (default: 10,000).
+    max_total_size_bytes : int | None
+        Maximum total size of all files to process in bytes. If ``None``, uses the default from config (default: 500 MB).
+    max_directory_depth : int | None
+        Maximum depth of directory traversal. If ``None``, uses the default from config (default: 20).
     include_patterns : str | set[str] | None
         Pattern or set of patterns specifying which files to include. If ``None``, all files are included.
     exclude_patterns : str | set[str] | None
@@ -159,6 +174,9 @@ def ingest(
         ingest_async(
             source=source,
             max_file_size=max_file_size,
+            max_files=max_files,
+            max_total_size_bytes=max_total_size_bytes,
+            max_directory_depth=max_directory_depth,
             include_patterns=include_patterns,
             exclude_patterns=exclude_patterns,
             branch=branch,

src/gitingest/ingestion.py

@@ -97,7 +97,7 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
         Statistics tracking object for the total file count and size.
 
     """
-    if limit_exceeded(stats, depth=node.depth):
+    if limit_exceeded(stats, depth=node.depth, query=query):
         return
 
     for sub_path in node.path.iterdir():
@@ -113,7 +113,7 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
             if sub_path.stat().st_size > query.max_file_size:
                 print(f"Skipping file {sub_path}: would exceed max file size limit")
                 continue
-            _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
+            _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path, query=query)
         elif sub_path.is_dir():
             child_directory_node = FileSystemNode(
                 name=sub_path.name,
@@ -167,7 +167,7 @@ def _process_symlink(path: Path, parent_node: FileSystemNode, stats: FileSystemS
     parent_node.file_count += 1
 
 
-def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStats, local_path: Path) -> None:
+def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStats, local_path: Path, query: IngestionQuery) -> None:
     """Process a file in the file system.
 
     This function checks the file's size, increments the statistics, and reads its content.
@@ -183,14 +183,16 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
         Statistics tracking object for the total file count and size.
     local_path : Path
         The base path of the repository or directory being processed.
+    query : IngestionQuery
+        The query object containing the limit configurations.
 
     """
-    if stats.total_files + 1 > MAX_FILES:
-        print(f"Maximum file limit ({MAX_FILES}) reached")
+    if stats.total_files + 1 > query.max_files:
+        print(f"Maximum file limit ({query.max_files}) reached")
         return
 
     file_size = path.stat().st_size
-    if stats.total_size + file_size > MAX_TOTAL_SIZE_BYTES:
+    if stats.total_size + file_size > query.max_total_size_bytes:
         print(f"Skipping file {path}: would exceed total size limit")
         return
 
@@ -212,7 +214,7 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
     parent_node.file_count += 1
 
 
-def limit_exceeded(stats: FileSystemStats, depth: int) -> bool:
+def limit_exceeded(stats: FileSystemStats, depth: int, query: IngestionQuery) -> bool:
     """Check if any of the traversal limits have been exceeded.
 
     This function checks if the current traversal has exceeded any of the configured limits:
@@ -224,23 +226,25 @@ def limit_exceeded(stats: FileSystemStats, depth: int) -> bool:
         Statistics tracking object for the total file count and size.
     depth : int
         The current depth of directory traversal.
+    query : IngestionQuery
+        The query object containing the limit configurations.
 
     Returns
     -------
     bool
         ``True`` if any limit has been exceeded, ``False`` otherwise.
 
     """
-    if depth > MAX_DIRECTORY_DEPTH:
-        print(f"Maximum depth limit ({MAX_DIRECTORY_DEPTH}) reached")
+    if depth > query.max_directory_depth:
+        print(f"Maximum depth limit ({query.max_directory_depth}) reached")
         return True
 
-    if stats.total_files >= MAX_FILES:
-        print(f"Maximum file limit ({MAX_FILES}) reached")
+    if stats.total_files >= query.max_files:
+        print(f"Maximum file limit ({query.max_files}) reached")
         return True  # TODO: end recursion
 
-    if stats.total_size >= MAX_TOTAL_SIZE_BYTES:
-        print(f"Maxumum total size limit ({MAX_TOTAL_SIZE_BYTES / 1024 / 1024:.1f}MB) reached")
+    if stats.total_size >= query.max_total_size_bytes:
+        print(f"Maxumum total size limit ({query.max_total_size_bytes / 1024 / 1024:.1f}MB) reached")
         return True  # TODO: end recursion
 
     return False

src/gitingest/query_parser.py

@@ -28,6 +28,9 @@ async def parse_query(
     source: str,
     *,
     max_file_size: int,
+    max_files: int | None = None,
+    max_total_size_bytes: int | None = None,
+    max_directory_depth: int | None = None,
     from_web: bool,
     include_patterns: set[str] | str | None = None,
     ignore_patterns: set[str] | str | None = None,
@@ -41,6 +44,12 @@ async def parse_query(
         The source URL or file path to parse.
     max_file_size : int
         The maximum file size in bytes to include.
+    max_files : int | None
+        The maximum number of files to process. If None, uses default from config.
+    max_total_size_bytes : int | None
+        The maximum total size of all files in bytes. If None, uses default from config.
+    max_directory_depth : int | None
+        The maximum depth of directory traversal. If None, uses default from config.
     from_web : bool
         Flag indicating whether the source is a web URL.
     include_patterns : set[str] | str | None
@@ -89,6 +98,9 @@ async def parse_query(
         branch=query.branch,
         commit=query.commit,
         max_file_size=max_file_size,
+        max_files=max_files if max_files is not None else query.max_files,
+        max_total_size_bytes=max_total_size_bytes if max_total_size_bytes is not None else query.max_total_size_bytes,
+        max_directory_depth=max_directory_depth if max_directory_depth is not None else query.max_directory_depth,
         ignore_patterns=ignore_patterns_set,
         include_patterns=parsed_include,
     )

src/gitingest/schemas/ingestion.py

@@ -7,7 +7,7 @@
 
 from pydantic import BaseModel, Field
 
-from gitingest.config import MAX_FILE_SIZE
+from gitingest.config import MAX_FILE_SIZE, MAX_FILES, MAX_TOTAL_SIZE_BYTES, MAX_DIRECTORY_DEPTH
 
 
 @dataclass
@@ -77,6 +77,12 @@ class IngestionQuery(BaseModel):  # pylint: disable=too-many-instance-attributes
         The tag of the repository.
     max_file_size : int
         The maximum file size to ingest (default: 10 MB).
+    max_files : int
+        The maximum number of files to process (default: 10,000).
+    max_total_size_bytes : int
+        The maximum total size of all files in bytes (default: 500 MB).
+    max_directory_depth : int
+        The maximum depth of directory traversal (default: 20).
     ignore_patterns : set[str]
         The patterns to ignore (default: ``set()``).
     include_patterns : set[str] | None
@@ -98,6 +104,9 @@ class IngestionQuery(BaseModel):  # pylint: disable=too-many-instance-attributes
     commit: str | None = None
     tag: str | None = None
     max_file_size: int = Field(default=MAX_FILE_SIZE)
+    max_files: int = Field(default=MAX_FILES)
+    max_total_size_bytes: int = Field(default=MAX_TOTAL_SIZE_BYTES)
+    max_directory_depth: int = Field(default=MAX_DIRECTORY_DEPTH)
     ignore_patterns: set[str] = set()  # TODO: ignore_patterns and include_patterns have the same type
     include_patterns: set[str] | None = None
     include_submodules: bool = False