fix

Author: filipchristiansen

src/gitingest/__init__.py

@@ -3,6 +3,5 @@
 from gitingest.clone import clone_repo
 from gitingest.entrypoint import ingest, ingest_async
 from gitingest.ingestion import ingest_query
-from gitingest.query_parser import parse_query
 
-__all__ = ["clone_repo", "ingest", "ingest_async", "ingest_query", "parse_query"]
+__all__ = ["clone_repo", "ingest", "ingest_async", "ingest_query"]

src/gitingest/clone.py

@@ -56,7 +56,7 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
         msg = "Repository not found. Make sure it is public or that you have provided a valid token."
         raise ValueError(msg)
 
-    commit = await resolve_commit(config, url=url, token=token)
+    commit = await resolve_commit(config, token=token)
 
     clone_cmd = ["git"]
     if token and is_github_host(url):
@@ -73,7 +73,7 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
 
     # Checkout the subpath if it is a partial clone
     if partial_clone:
-        await checkout_partial_clone(config, token)
+        await checkout_partial_clone(config, token=token)
 
     git = create_git_command(["git"], local_path, url, token)
 

src/gitingest/entrypoint.py

@@ -8,14 +8,21 @@
 import warnings
 from contextlib import asynccontextmanager
 from pathlib import Path
-from typing import AsyncGenerator
+from typing import TYPE_CHECKING, AsyncGenerator
+from urllib.parse import urlparse
 
 from gitingest.clone import clone_repo
 from gitingest.config import MAX_FILE_SIZE
 from gitingest.ingestion import ingest_query
-from gitingest.query_parser import IngestionQuery, parse_query
+from gitingest.query_parser import parse_local_dir_path, parse_remote_repo
 from gitingest.utils.auth import resolve_token
+from gitingest.utils.compat_func import removesuffix
 from gitingest.utils.ignore_patterns import load_ignore_patterns
+from gitingest.utils.pattern_utils import process_patterns
+from gitingest.utils.query_parser_utils import KNOWN_GIT_HOSTS
+
+if TYPE_CHECKING:
+    from gitingest.schemas import IngestionQuery
 
 
 async def ingest_async(
@@ -74,23 +81,28 @@ async def ingest_async(
     """
     token = resolve_token(token)
 
-    query: IngestionQuery = await parse_query(
-        source=source,
-        max_file_size=max_file_size,
-        from_web=False,
+    source = removesuffix(source.strip(), ".git")
+
+    # Determine the parsing method based on the source type
+    if urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
+        # We either have a full URL or a domain-less slug
+        query = await parse_remote_repo(source, token=token)
+        query.include_submodules = include_submodules
+        _override_branch_and_tag(query, branch=branch, tag=tag)
+
+    else:
+        # Local path scenario
+        query = parse_local_dir_path(source)
+
+    query.max_file_size = max_file_size
+    query.ignore_patterns, query.include_patterns = process_patterns(
+        exclude_patterns=exclude_patterns,
         include_patterns=include_patterns,
-        ignore_patterns=exclude_patterns,
-        token=token,
     )
 
     if not include_gitignored:
         _apply_gitignores(query)
 
-    if query.url:
-        _override_branch_and_tag(query, branch=branch, tag=tag)
-
-    query.include_submodules = include_submodules
-
     async with _clone_repo_if_remote(query, token=token):
         summary, tree, content = ingest_query(query)
         await _write_output(tree, content=content, target=output)

src/gitingest/ingestion.py

@@ -11,7 +11,7 @@
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 
 if TYPE_CHECKING:
-    from gitingest.query_parser import IngestionQuery
+    from gitingest.schemas import IngestionQuery
 
 
 def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:

src/gitingest/output_formatter.py

@@ -10,7 +10,7 @@
 from gitingest.utils.compat_func import readlink
 
 if TYPE_CHECKING:
-    from gitingest.query_parser import IngestionQuery
+    from gitingest.schemas import IngestionQuery
 
 _TOKEN_THRESHOLDS: list[tuple[int, str]] = [
     (1_000_000, "M"),
@@ -84,6 +84,8 @@ def _create_summary_prefix(query: IngestionQuery, *, single_file: bool = False)
 
     if query.commit:
         parts.append(f"Commit: {query.commit}")
+    elif query.tag:
+        parts.append(f"Tag: {query.tag}")
     elif query.branch and query.branch not in ("main", "master"):
         parts.append(f"Branch: {query.branch}")
 

src/gitingest/query_parser.py

@@ -2,101 +2,24 @@
 
 from __future__ import annotations
 
-import re
 import uuid
 import warnings
 from pathlib import Path
 from urllib.parse import unquote, urlparse
 
 from gitingest.config import TMP_BASE_PATH
 from gitingest.schemas import IngestionQuery
-from gitingest.utils.exceptions import InvalidPatternError
 from gitingest.utils.git_utils import check_repo_exists, fetch_remote_branches_or_tags
-from gitingest.utils.ignore_patterns import DEFAULT_IGNORE_PATTERNS
 from gitingest.utils.query_parser_utils import (
     KNOWN_GIT_HOSTS,
     _get_user_and_repo_from_path,
     _is_valid_git_commit_hash,
-    _is_valid_pattern,
     _validate_host,
     _validate_url_scheme,
 )
 
 
-async def parse_query(
-    source: str,
-    *,
-    max_file_size: int,
-    from_web: bool,
-    include_patterns: set[str] | str | None = None,
-    ignore_patterns: set[str] | str | None = None,
-    token: str | None = None,
-) -> IngestionQuery:
-    """Parse the input source to extract details for the query and process the include and ignore patterns.
-
-    Parameters
-    ----------
-    source : str
-        The source URL or file path to parse.
-    max_file_size : int
-        The maximum file size in bytes to include.
-    from_web : bool
-        Flag indicating whether the source is a web URL.
-    include_patterns : set[str] | str | None
-        Patterns to include. Can be a set of strings or a single string.
-    ignore_patterns : set[str] | str | None
-        Patterns to ignore. Can be a set of strings or a single string.
-    token : str | None
-        GitHub personal access token (PAT) for accessing private repositories.
-
-    Returns
-    -------
-    IngestionQuery
-        A dataclass object containing the parsed details of the repository or file path.
-
-    """
-    if source.endswith(".git"):
-        source = source[:-4]
-
-    # Determine the parsing method based on the source type
-    if from_web or urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
-        # We either have a full URL or a domain-less slug
-        query = await _parse_remote_repo(source, token=token)
-    else:
-        # Local path scenario
-        query = _parse_local_dir_path(source)
-
-    # Combine default ignore patterns + custom patterns
-    ignore_patterns_set = DEFAULT_IGNORE_PATTERNS.copy()
-    if ignore_patterns:
-        ignore_patterns_set.update(_parse_patterns(ignore_patterns))
-
-    # Process include patterns and override ignore patterns accordingly
-    if include_patterns:
-        parsed_include = _parse_patterns(include_patterns)
-        # Override ignore patterns with include patterns
-        ignore_patterns_set = set(ignore_patterns_set) - set(parsed_include)
-    else:
-        parsed_include = None
-
-    return IngestionQuery(
-        user_name=query.user_name,
-        repo_name=query.repo_name,
-        url=query.url,
-        subpath=query.subpath,
-        local_path=query.local_path,
-        slug=query.slug,
-        id=query.id,
-        type=query.type,
-        branch=query.branch,
-        commit=query.commit,
-        max_file_size=max_file_size,
-        ignore_patterns=ignore_patterns_set,
-        include_patterns=parsed_include,
-    )
-
-
-async def _parse_remote_repo(source: str, token: str | None = None) -> IngestionQuery:
+async def parse_remote_repo(source: str, token: str | None = None) -> IngestionQuery:
     """Parse a repository URL into a structured query dictionary.
 
     If source is:
@@ -146,7 +69,8 @@ async def _parse_remote_repo(source: str, token: str | None = None) -> Ingestion
     local_path = TMP_BASE_PATH / _id / slug
     url = f"https://{host}/{user_name}/{repo_name}"
 
-    parsed = IngestionQuery(
+    query = IngestionQuery(
+        host=host,
         user_name=user_name,
         repo_name=repo_name,
         url=url,
@@ -158,57 +82,76 @@ async def _parse_remote_repo(source: str, token: str | None = None) -> Ingestion
     remaining_parts = parsed_url.path.strip("/").split("/")[2:]
 
     if not remaining_parts:
-        return parsed
+        return query
 
     possible_type = remaining_parts.pop(0)  # e.g. 'issues', 'pull', 'tree', 'blob'
 
     # If no extra path parts, just return
     if not remaining_parts:
-        return parsed
+        return query
 
     # If this is an issues page or pull requests, return early without processing subpath
     # TODO: Handle issues and pull requests
     if remaining_parts and possible_type in {"issues", "pull"}:
         msg = f"Warning: Issues and pull requests are not yet supported: {url}. Returning repository root."
         warnings.warn(msg, RuntimeWarning, stacklevel=2)
-        return parsed
+        return query
 
     if possible_type not in {"tree", "blob"}:
         # TODO: Handle other types
         msg = f"Warning: Type '{possible_type}' is not yet supported: {url}. Returning repository root."
         warnings.warn(msg, RuntimeWarning, stacklevel=2)
-        return parsed
+        return query
 
-    parsed.type = possible_type  # 'tree' or 'blob'
+    query.type = possible_type
 
     # Commit, branch, or tag
     commit_or_branch_or_tag = remaining_parts[0]
     if _is_valid_git_commit_hash(commit_or_branch_or_tag):  # Commit
-        parsed.commit = commit_or_branch_or_tag
+        query.commit = commit_or_branch_or_tag
         remaining_parts.pop(0)  # Consume the commit hash
     else:  # Branch or tag
         # Try to resolve a tag
-        parsed.tag = await _configure_branch_or_tag(
+        query.tag = await _configure_branch_or_tag(
             remaining_parts,
             url=url,
             ref_type="tags",
             token=token,
         )
 
         # If no tag found, try to resolve a branch
-        if not parsed.tag:
-            parsed.branch = await _configure_branch_or_tag(
+        if not query.tag:
+            query.branch = await _configure_branch_or_tag(
                 remaining_parts,
                 url=url,
                 ref_type="branches",
                 token=token,
             )
 
     # Only configure subpath if we have identified a commit, branch, or tag.
-    if remaining_parts and (parsed.commit or parsed.branch or parsed.tag):
-        parsed.subpath += "/".join(remaining_parts)
+    if remaining_parts and (query.commit or query.branch or query.tag):
+        query.subpath += "/".join(remaining_parts)
+
+    return query
 
-    return parsed
+
+def parse_local_dir_path(path_str: str) -> IngestionQuery:
+    """Parse the given file path into a structured query dictionary.
+
+    Parameters
+    ----------
+    path_str : str
+        The file path to parse.
+
+    Returns
+    -------
+    IngestionQuery
+        A dictionary containing the parsed details of the file path.
+
+    """
+    path_obj = Path(path_str).resolve()
+    slug = path_obj.name if path_str == "." else path_str.strip("/")
+    return IngestionQuery(local_path=path_obj, slug=slug, id=str(uuid.uuid4()))
 
 
 async def _configure_branch_or_tag(
@@ -272,69 +215,6 @@ async def _configure_branch_or_tag(
     return None
 
 
-def _parse_patterns(pattern: set[str] | str) -> set[str]:
-    """Parse and validate file/directory patterns for inclusion or exclusion.
-
-    Takes either a single pattern string or set of pattern strings and processes them into a normalized list.
-    Patterns are split on commas and spaces, validated for allowed characters, and normalized.
-
-    Parameters
-    ----------
-    pattern : set[str] | str
-        Pattern(s) to parse - either a single string or set of strings
-
-    Returns
-    -------
-    set[str]
-        A set of normalized patterns.
-
-    Raises
-    ------
-    InvalidPatternError
-        If any pattern contains invalid characters. Only alphanumeric characters,
-        dash (-), underscore (_), dot (.), forward slash (/), plus (+), and
-        asterisk (*) are allowed.
-
-    """
-    patterns = pattern if isinstance(pattern, set) else {pattern}
-
-    parsed_patterns: set[str] = set()
-    for p in patterns:
-        parsed_patterns = parsed_patterns.union(set(re.split(",| ", p)))
-
-    # Remove empty string if present
-    parsed_patterns = parsed_patterns - {""}
-
-    # Normalize Windows paths to Unix-style paths
-    parsed_patterns = {p.replace("\\", "/") for p in parsed_patterns}
-
-    # Validate and normalize each pattern
-    for p in parsed_patterns:
-        if not _is_valid_pattern(p):
-            raise InvalidPatternError(p)
-
-    return parsed_patterns
-
-
-def _parse_local_dir_path(path_str: str) -> IngestionQuery:
-    """Parse the given file path into a structured query dictionary.
-
-    Parameters
-    ----------
-    path_str : str
-        The file path to parse.
-
-    Returns
-    -------
-    IngestionQuery
-        A dictionary containing the parsed details of the file path.
-
-    """
-    path_obj = Path(path_str).resolve()
-    slug = path_obj.name if path_str == "." else path_str.strip("/")
-    return IngestionQuery(local_path=path_obj, slug=slug, id=str(uuid.uuid4()))
-
-
 async def try_domains_for_user_and_repo(user_name: str, repo_name: str, token: str | None = None) -> str:
     """Attempt to find a valid repository host for the given ``user_name`` and ``repo_name``.