From 2cc274f56b5f49ec40e7fffe0d225fdc23c78f7a Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 9 Sep 2026 16:43:20 -0300 Subject: [PATCH 1/3] feat(backend): add commits and commit_parents schema (#2089) * Add `Commits` model: unique `git_commit_hash`, nullable author/committer/subject/message/`fetched_from_url` * Add `CommitParents` model: FKs to `commits.id`, `ord` with 0 = first parent, unique `(commit_id, ord)` * Add migration `0021_commits_and_commit_parents` (`commits`, `commit_parents` tables and indexes) * Join from `checkouts.git_commit_hash` without new checkout columns Signed-off-by: Alan Peixinho --- .../0021_commits_and_commit_parents.py | 69 +++++++++++++++++++ backend/kernelCI_app/models.py | 49 +++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 backend/kernelCI_app/migrations/0021_commits_and_commit_parents.py diff --git a/backend/kernelCI_app/migrations/0021_commits_and_commit_parents.py b/backend/kernelCI_app/migrations/0021_commits_and_commit_parents.py new file mode 100644 index 000000000..db1d7fb03 --- /dev/null +++ b/backend/kernelCI_app/migrations/0021_commits_and_commit_parents.py @@ -0,0 +1,69 @@ +# Generated by Django 5.2.16 on 2026-09-09 19:33 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0020_add_labs_fk_indexes"), + ] + + operations = [ + migrations.CreateModel( + name="Commits", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("git_commit_hash", models.TextField(unique=True)), + ("author_name", models.TextField(blank=True, null=True)), + ("author_email", models.TextField(blank=True, null=True)), + ("author_date", models.DateTimeField(blank=True, null=True)), + ("committer_name", models.TextField(blank=True, null=True)), + ("committer_email", models.TextField(blank=True, null=True)), + ("committer_date", models.DateTimeField(blank=True, null=True)), + ("subject", models.TextField(blank=True, null=True)), + ("message", models.TextField(blank=True, null=True)), + ("fetched_from_url", models.TextField(blank=True, null=True)), + ], + options={ + "db_table": "commits", + }, + ), + migrations.CreateModel( + name="CommitParents", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("ord", models.SmallIntegerField()), + ( + "commit", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="parent_edges", + to="kernelCI_app.commits", + ), + ), + ( + "parent", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="child_edges", + to="kernelCI_app.commits", + ), + ), + ], + options={ + "db_table": "commit_parents", + "indexes": [ + models.Index(fields=["parent"], name="commit_parents_parent_id"), + ], + "constraints": [ + models.UniqueConstraint( + fields=("commit", "ord"), + name="commit_parents_commit_ord", + ), + ], + }, + ), + ] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index f3e259dc9..7d48e6372 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -109,6 +109,55 @@ class Meta: ] +class Commits(models.Model): + id = models.AutoField(primary_key=True) + git_commit_hash = models.TextField(unique=True) + author_name = models.TextField(blank=True, null=True) + author_email = models.TextField(blank=True, null=True) + author_date = models.DateTimeField(blank=True, null=True) + committer_name = models.TextField(blank=True, null=True) + committer_email = models.TextField(blank=True, null=True) + committer_date = models.DateTimeField(blank=True, null=True) + subject = models.TextField(blank=True, null=True) + message = models.TextField(blank=True, null=True) + fetched_from_url = models.TextField(blank=True, null=True) + + class Meta: + db_table = "commits" + + def __str__(self) -> str: + return self.git_commit_hash + + +class CommitParents(models.Model): + id = models.AutoField(primary_key=True) + commit = models.ForeignKey( + Commits, + on_delete=models.CASCADE, + related_name="parent_edges", + db_index=False, + ) + parent = models.ForeignKey( + Commits, + on_delete=models.CASCADE, + related_name="child_edges", + db_index=False, + ) + ord = models.SmallIntegerField() + + class Meta: + db_table = "commit_parents" + constraints = [ + models.UniqueConstraint( + fields=["commit", "ord"], + name="commit_parents_commit_ord", + ), + ] + indexes = [ + models.Index(fields=["parent"], name="commit_parents_parent_id"), + ] + + class Builds(models.Model): field_timestamp = models.DateTimeField( db_column="_timestamp", blank=True, null=True From 7159f0875054c45de6f53a65933d4e21248099e3 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 9 Sep 2026 18:06:44 -0300 Subject: [PATCH 2/3] feat(backend): snapshot and restore commits in update_db Dump commits and commit_parents in full so git ancestry survives restore. Signed-off-by: Alan Peixinho --- backend/docs/update_db command.md | 12 +- .../management/commands/update_db.py | 128 +++++++++++++++++- 2 files changed, 134 insertions(+), 6 deletions(-) diff --git a/backend/docs/update_db command.md b/backend/docs/update_db command.md index 8acf81730..966b2038b 100644 --- a/backend/docs/update_db command.md +++ b/backend/docs/update_db command.md @@ -1,6 +1,6 @@ # update_db Command Documentation -The `update_db` command migrates data from the default database (kcidb) to the dashboard_db database within a specified time interval. All tables are updated by default, but you can select a specific one as well. +The `update_db` command snapshots dashboard tables to a `.tar.gz` and restores them. Most tables are limited to `--start-interval` / `--end-interval`. `commits` and `commit_parents` are copied in full for now (no time filter), so git ancestry is not cut when a parent has no checkout in the window. A later change may slice those tables. The migration preserves foreign key constraints. For example, if a test A references a build B in kcidb, but the build B doesn't exist in dashboard_db, then the test A will not be inserted in dashboard_db. @@ -8,13 +8,13 @@ The migration preserves foreign key constraints. For example, if a test A refere ### Required Parameters -- `--start-interval`: Start interval for filtering data (format: 'x days' or 'x hours'). The format follows the SQL filtering format. -- `--end-interval`: End interval for filtering data (format: 'x days' or 'x hours'). The format follows the SQL filtering format. +- `--start-interval`: Start interval for filtering data (format: 'x days' or 'x hours'). The format follows the SQL filtering format. Does not apply to `commits` or `commit_parents`. +- `--end-interval`: End interval for filtering data (format: 'x days' or 'x hours'). The format follows the SQL filtering format. Does not apply to `commits` or `commit_parents`. ### Optional Parameters - `--table`: Limit data copy to a specific table - - Valid options: `issues`, `checkouts`, `builds`, `tests`, `incidents` + - Valid options: `issues`, `checkouts`, `commits`, `commit_parents`, `builds`, `tests`, `incidents`, `latest_checkout`, `hardware_status`, `tree_listing`, `tree_tests_rollup` - If not provided, data from all tables will be copied - `--related-data-only`: Limits the selected data to data where the foreign key constraint is not broken. - Default: False. @@ -35,7 +35,7 @@ python manage.py update_db --start-interval "1 days" --end-interval "0 days" --t ## Migration Process -1. **Data Selection**: Selects records from the default database within the specified time range +1. **Data Selection**: Selects records from the default database within the specified time range, except `commits` and `commit_parents` (full table) 2. **Relationship Validation**: Ensures foreign key constraints are maintained 3. **Data Insertion**: Inserts valid data into the dashboard db 4. **Conflict Resolution**: Uses `ignore_conflicts=True` to handle duplicate records @@ -44,6 +44,8 @@ python manage.py update_db --start-interval "1 days" --end-interval "0 days" --t - Migration preserves JSON fields by parsing them appropriately - The skipped rows count are related to rows which didn't have relationships in dashboard_db. The processed rows count are related to the remaining rows that were selected but not skipped (even if they were inserted or had a conflict, which is how django returns the `bulk_create` result) +- `commits` and `commit_parents` are not filtered by time or origin. The full git graph is dumped so ancestors without a checkout in the window are kept. Time slicing may be added later. +- Surrogate ids are kept as in the source. Restore runs `setval` to `MAX(id)` so the next insert does not collide. ## Performance Considerations diff --git a/backend/kernelCI_app/management/commands/update_db.py b/backend/kernelCI_app/management/commands/update_db.py index 75b364a25..3be7d2dfd 100644 --- a/backend/kernelCI_app/management/commands/update_db.py +++ b/backend/kernelCI_app/management/commands/update_db.py @@ -18,6 +18,8 @@ from kernelCI_app.models import ( Builds, Checkouts, + CommitParents, + Commits, HardwareStatus, Incidents, Issues, @@ -143,7 +145,8 @@ def _invalid_table_error(self, table: str) -> str: return ( f"Unknown table '{table}'.\n" "\tValid options are: issues, checkouts, builds, tests, incidents, " - "latest_checkout, hardware_status, tree_listing, tree_tests_rollup." + "commits, commit_parents, latest_checkout, hardware_status, " + "tree_listing, tree_tests_rollup." ) def handle(self, *args, command, **options): @@ -223,6 +226,8 @@ def snapshot(self, table, snapshot_filepath: Path): case None: self.snapshot_issues() self.snapshot_checkouts() + self.snapshot_commits() + self.snapshot_commit_parents() self.snapshot_builds() self.snapshot_tests() self.snapshot_incidents() @@ -234,6 +239,10 @@ def snapshot(self, table, snapshot_filepath: Path): self.snapshot_issues() case "checkouts": self.snapshot_checkouts() + case "commits": + self.snapshot_commits() + case "commit_parents": + self.snapshot_commit_parents() case "builds": self.snapshot_builds() case "tests": @@ -262,6 +271,8 @@ def snapshot(self, table, snapshot_filepath: Path): def restore(self, snapshot_filepath: Path): self.snapshot_archive = tarfile.open(snapshot_filepath, "r:*") try: + self.restore_commits() + self.restore_commit_parents() self.restore_checkouts() self.restore_builds() self.restore_issues() @@ -304,6 +315,14 @@ def get_related_data( return related_ids, related_condition + def sync_id_sequence(self, table: str) -> None: + with connections["default"].cursor() as cursor: + cursor.execute( + f"SELECT setval(pg_get_serial_sequence(%s, 'id')," + f" COALESCE((SELECT MAX(id) FROM {table}), 1))", + [table], + ) + # ISSUES ######################################## def select_issues_data(self) -> list[tuple]: query = f""" @@ -461,6 +480,113 @@ def restore_checkouts(self) -> None: self.insert_checkouts_data(records) self.stdout.write("Checkouts migration completed") + # COMMITS ######################################## + def select_commits_data(self) -> Generator[list[tuple], None, None]: + query = """ + SELECT id, git_commit_hash, author_name, author_email, author_date, + committer_name, committer_email, committer_date, subject, + message, fetched_from_url + FROM commits + ORDER BY id + """ + with connections["default"].cursor() as cursor: + cursor.execute(query) + while batch := cursor.fetchmany(SELECT_BATCH_SIZE): + yield batch + + def insert_commits_data(self, records: list[tuple]) -> int: + original_commits = [ + Commits( + id=record[0], + git_commit_hash=record[1], + author_name=record[2] or None, + author_email=record[3] or None, + author_date=parse_datetime(record[4]) if record[4] else None, + committer_name=record[5] or None, + committer_email=record[6] or None, + committer_date=parse_datetime(record[7]) if record[7] else None, + subject=record[8] or None, + message=record[9] or None, + fetched_from_url=record[10] or None, + ) + for record in records + ] + migrated_commits = Commits.objects.bulk_create( + original_commits, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) + self.sync_id_sequence("commits") + total_inserted = len(migrated_commits) + self.stdout.write(f"Processed {total_inserted} Commits records") + return total_inserted + + def snapshot_commits(self) -> None: + with SpooledTemporaryFile(mode="w+b", max_size=MAX_MEMORY_BUFFER_BYTES) as file: + self.stdout.write("\nMigrating Commits...") + for record_batch in self.select_commits_data(): + self.insert_records(file, "commits", record_batch) + self.add_file_to_snapshot(file, "commits") + self.stdout.write("Commits migration completed") + + def restore_commits(self) -> None: + with TextIOWrapper(self.snapshot_archive.extractfile("commits.csv")) as file: + self.stdout.write("\nMigrating Commits...") + reader = csv.reader(file) + while records := self.read_records(reader, max_rows=SELECT_BATCH_SIZE): + self.insert_commits_data(records) + self.stdout.write("Commits migration completed") + + # COMMIT PARENTS ######################################## + def select_commit_parents_data(self) -> Generator[list[tuple], None, None]: + query = """ + SELECT id, commit_id, parent_id, ord + FROM commit_parents + ORDER BY id + """ + with connections["default"].cursor() as cursor: + cursor.execute(query) + while batch := cursor.fetchmany(SELECT_BATCH_SIZE): + yield batch + + def insert_commit_parents_data(self, records: list[tuple]) -> int: + original_parents = [ + CommitParents( + id=record[0], + commit_id=record[1], + parent_id=record[2], + ord=record[3], + ) + for record in records + ] + migrated_parents = CommitParents.objects.bulk_create( + original_parents, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) + self.sync_id_sequence("commit_parents") + total_inserted = len(migrated_parents) + self.stdout.write(f"Processed {total_inserted} CommitParents records") + return total_inserted + + def snapshot_commit_parents(self) -> None: + with SpooledTemporaryFile(mode="w+b", max_size=MAX_MEMORY_BUFFER_BYTES) as file: + self.stdout.write("\nMigrating CommitParents...") + for record_batch in self.select_commit_parents_data(): + self.insert_records(file, "commit_parents", record_batch) + self.add_file_to_snapshot(file, "commit_parents") + self.stdout.write("CommitParents migration completed") + + def restore_commit_parents(self) -> None: + with TextIOWrapper( + self.snapshot_archive.extractfile("commit_parents.csv") + ) as file: + self.stdout.write("\nMigrating CommitParents...") + reader = csv.reader(file) + while records := self.read_records(reader, max_rows=SELECT_BATCH_SIZE): + self.insert_commit_parents_data(records) + self.stdout.write("CommitParents migration completed") + # BUILDS ######################################## def select_builds_data(self) -> list[tuple]: related_checkout_ids, related_condition = self.get_related_data( From 27eed44d2418cbfa52e177a41b560b657a0b7003 Mon Sep 17 00:00:00 2001 From: Felipe Bergamin Date: Fri, 11 Sep 2026 15:41:03 -0300 Subject: [PATCH 3/3] feat(backend): parse and fetch single-commit metadata (#2090) Add a SHA-only helper for commit author/parents so later sync can reuse parsing without duplicating git format handling. Signed-off-by: Felipe Bergamin --- .env.backend.example | 2 + .env.example | 2 + backend/Dockerfile | 1 + backend/kernelCI/settings.py | 4 + backend/kernelCI_app/helpers/gitCommit.py | 338 ++++++++++++++++++ .../helpers/fixtures/git_commit_data.py | 20 ++ .../tests/unitTests/helpers/gitCommit_test.py | 248 +++++++++++++ 7 files changed, 615 insertions(+) create mode 100644 backend/kernelCI_app/helpers/gitCommit.py create mode 100644 backend/kernelCI_app/tests/unitTests/helpers/fixtures/git_commit_data.py create mode 100644 backend/kernelCI_app/tests/unitTests/helpers/gitCommit_test.py diff --git a/.env.backend.example b/.env.backend.example index bd61973aa..e8338d89f 100644 --- a/.env.backend.example +++ b/.env.backend.example @@ -25,6 +25,8 @@ PROMETHEUS_MULTIPROC_DIR=/tmp/metrics INGESTER_METRICS_PORT=8002 BACKEND_VOLUME_DIR=/volume_data +# Ephemeral git clones for SHA-only commit fetch (#2090). Prefer tmpfs. +GIT_SCRATCH_DIR=/dev/shm/kernelci-git-scratch ## Variables used for the notifications command. Check docs/notifications.md # EMAIL_HOST_USER="youruser@host" # (optional) diff --git a/.env.example b/.env.example index ccdf18e3a..82d66bb94 100644 --- a/.env.example +++ b/.env.example @@ -106,6 +106,8 @@ HEALTHCHECK_ID_NOTIFICATIONS_SUMMARY_MAESTRO= # Backend Volume # ----------------------------------------------------------------------------- BACKEND_VOLUME_DIR=/volume_data +# Ephemeral git clones for SHA-only commit fetch (#2090). Prefer tmpfs. +GIT_SCRATCH_DIR=/dev/shm/kernelci-git-scratch # ----------------------------------------------------------------------------- # Ingester (only needed with --profile=with_commands) diff --git a/backend/Dockerfile b/backend/Dockerfile index c7474f5ff..d6e454f10 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -19,6 +19,7 @@ RUN apk update \ libpq-dev \ postgresql \ curl \ + git \ && python3 -m venv $POETRY_HOME \ && $POETRY_HOME/bin/pip install poetry~=2.4.0 \ && ln -s $POETRY_HOME/bin/poetry /bin/poetry diff --git a/backend/kernelCI/settings.py b/backend/kernelCI/settings.py index 73d157270..531e8f783 100644 --- a/backend/kernelCI/settings.py +++ b/backend/kernelCI/settings.py @@ -268,6 +268,10 @@ def get_json_env_var(name, default): # https://docs.djangoproject.com/en/5.0/ref/settings/#databases BACKEND_VOLUME_DIR = os.environ.get("BACKEND_VOLUME_DIR", "/volume_data") +# Throwaway git dirs for one-shot SHA fetches (#2090). Prefer tmpfs (e.g. /dev/shm). +GIT_SCRATCH_DIR = os.environ.get( + "GIT_SCRATCH_DIR", "/dev/shm/kernelci-git-scratch" +) DATABASE_ROUTERS = ["kernelCI_app.routers.databaseRouter.DatabaseRouter"] diff --git a/backend/kernelCI_app/helpers/gitCommit.py b/backend/kernelCI_app/helpers/gitCommit.py new file mode 100644 index 000000000..c9fb4e783 --- /dev/null +++ b/backend/kernelCI_app/helpers/gitCommit.py @@ -0,0 +1,338 @@ +"""Parse git commit metadata. Optional one-shot SHA fetch; no DB writes. + +Used as a fallback for hashes the mirrored-tree sync job (#2109) cannot +cover. Parse against an existing repo path so that job can reuse this +instead of duplicating commit-format parsing. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import urlparse + +from django.conf import settings + +from kernelCI_app.models import Checkouts + +FETCH_TIMEOUT_SECONDS = 60 +# One commit object is tiny; a full kernel history pack is hundreds of MB. +MAX_EPHEMERAL_PACK_BYTES = 2 * 1024 * 1024 +_IDENT_RE = re.compile(r"^([^<]*?) <([^>]*)> (\d+) ([+-]\d{4})$") +_GIT_ENV = { + "GIT_TERMINAL_PROMPT": "0", + "GCM_INTERACTIVE": "never", +} + + +class CommitMetadataError(Exception): + """Typed failure from parse or one-shot fetch. No metadata returned.""" + + +class InvalidGitUrlError(CommitMetadataError): + pass + + +class MissingGitUrlError(CommitMetadataError): + pass + + +class FetchFailedError(CommitMetadataError): + pass + + +class OversizedPackError(CommitMetadataError): + pass + + +class UnexpectedFetchObjectsError(CommitMetadataError): + pass + + +class CommitParseError(CommitMetadataError): + pass + + +@dataclass(frozen=True) +class CommitMetadata: + git_commit_hash: str + author_name: str | None + author_email: str | None + author_date: datetime | None + committer_name: str | None + committer_email: str | None + committer_date: datetime | None + subject: str | None + message: str | None + parent_hashes: tuple[str, ...] + + +def sanitize_git_url(git_url: str | None) -> str | None: + """Treeproof-style cleanup. Returns None for malformed URLs; never raises.""" + if not isinstance(git_url, str) or not git_url.strip(): + return None + + raw = git_url.strip().rstrip("/") + if "://" not in raw: + return None + + parsed = urlparse(raw) + if not parsed.scheme: + return None + if parsed.scheme != "file" and not parsed.netloc: + return None + if not [segment for segment in parsed.path.split("/") if segment]: + return None + return raw + + +def resolve_checkout_git_url(git_commit_hash: str) -> str | None: + """Pick a fetch URL from checkouts for this hash. Prefer maestro / git.kernel.org.""" + rows = ( + Checkouts.objects.filter( + git_commit_hash=git_commit_hash, + git_repository_url__isnull=False, + ) + .values_list("origin", "git_repository_url") + .distinct() + ) + + best_url: str | None = None + best_rank: tuple[int, str] | None = None + for origin, url in rows: + cleaned = sanitize_git_url(url) + if cleaned is None: + continue + rank = _url_preference(origin or "", cleaned) + if best_rank is None or rank < best_rank: + best_rank = rank + best_url = cleaned + return best_url + + +def parse_commit_object(*, raw: str, git_commit_hash: str) -> CommitMetadata: + """Parse a raw commit object (`git cat-file -p`). No git, no fetch.""" + if not git_commit_hash: + raise CommitParseError("missing git_commit_hash") + + headers, separator, message = raw.partition("\n\n") + if not separator: + raise CommitParseError("commit object has no header/message separator") + + parent_hashes: list[str] = [] + author = (None, None, None) + committer = (None, None, None) + for line in _header_lines(headers): + if line.startswith("parent "): + parent_hashes.append(line.removeprefix("parent ").strip()) + elif line.startswith("author "): + author = _parse_ident(line.removeprefix("author ")) + elif line.startswith("committer "): + committer = _parse_ident(line.removeprefix("committer ")) + + subject = message.split("\n", 1)[0] if message else None + if subject == "": + subject = None + + return CommitMetadata( + git_commit_hash=git_commit_hash, + author_name=author[0], + author_email=author[1], + author_date=author[2], + committer_name=committer[0], + committer_email=committer[1], + committer_date=committer[2], + subject=subject, + message=message if message else None, + parent_hashes=tuple(parent_hashes), + ) + + +def parse_commit(*, repo_path: str, git_commit_hash: str) -> CommitMetadata: + """Read one commit from an existing repo. Does not fetch.""" + try: + full_hash = ( + _git( + Path(repo_path), + "rev-parse", + "--verify", + f"{git_commit_hash}^{{commit}}", + ) + .decode() + .strip() + ) + raw = _git(Path(repo_path), "cat-file", "-p", full_hash).decode( + "utf-8", errors="replace" + ) + except FetchFailedError as exc: + raise CommitParseError(str(exc)) from exc + return parse_commit_object(raw=raw, git_commit_hash=full_hash) + + +def fetch_commit_metadata( + git_commit_hash: str, + url: str | None = None, +) -> CommitMetadata: + """Fetch a single SHA into a throwaway bare repo, parse it, wipe the repo. + + Not the fill path for `commits`: no ancestry, no parent-object fetch. + """ + remote_url = _resolve_fetch_url(git_commit_hash, url) + scratch = Path(settings.GIT_SCRATCH_DIR) + scratch.mkdir(parents=True, exist_ok=True) + repo_dir = Path(tempfile.mkdtemp(prefix="commit-fetch-", dir=scratch)) + try: + _git(repo_dir, "init", "--bare") + _git(repo_dir, "remote", "add", "origin", remote_url) + _git( + repo_dir, + "fetch", + "--no-tags", + "--depth=1", + "--filter=tree:0", + "origin", + git_commit_hash, + timeout=FETCH_TIMEOUT_SECONDS, + ) + assert_single_commit_fetch(repo_dir) + return parse_commit(repo_path=str(repo_dir), git_commit_hash=git_commit_hash) + except CommitMetadataError: + raise + except Exception as exc: + raise FetchFailedError( + f"fetch {git_commit_hash} from {remote_url} failed: {exc}" + ) from exc + finally: + shutil.rmtree(repo_dir, ignore_errors=True) + + +def assert_single_commit_fetch(repo_dir: Path) -> None: + """Fail if the remote ignored shallow/filter and sent a full or treeful pack.""" + pack_bytes = _pack_bytes(repo_dir) + if pack_bytes > MAX_EPHEMERAL_PACK_BYTES: + raise OversizedPackError( + f"ephemeral fetch pack is {pack_bytes} bytes (max {MAX_EPHEMERAL_PACK_BYTES})" + ) + + counts = _object_type_counts(repo_dir) + if counts.get("tree", 0) or counts.get("blob", 0): + raise UnexpectedFetchObjectsError( + f"ephemeral fetch included trees/blobs: {counts}" + ) + if counts.get("commit", 0) != 1: + raise UnexpectedFetchObjectsError( + f"ephemeral fetch must contain exactly one commit: {counts}" + ) + + +def _resolve_fetch_url(git_commit_hash: str, url: str | None) -> str: + if url is not None: + cleaned = sanitize_git_url(url) + if cleaned is None: + raise InvalidGitUrlError(f"malformed git url: {url!r}") + return cleaned + + resolved = resolve_checkout_git_url(git_commit_hash) + if resolved is None: + raise MissingGitUrlError(f"no usable git url for {git_commit_hash}") + return resolved + + +def _url_preference(origin: str, url: str) -> tuple[int, str]: + host = urlparse(url).netloc.lower() + kernel_org = "git.kernel.org" in host + maestro = origin == "maestro" + if maestro and kernel_org: + tier = 0 + elif kernel_org: + tier = 1 + elif maestro: + tier = 2 + else: + tier = 3 + return (tier, url) + + +def _header_lines(headers: str): + for line in headers.split("\n"): + if line.startswith(" "): + continue + yield line + + +def _parse_ident( + ident: str, +) -> tuple[str | None, str | None, datetime | None]: + match = _IDENT_RE.match(ident.strip()) + if match is None: + return (None, None, None) + name, email, unix, tz = match.groups() + name = name.strip() or None + email = email.strip() or None + return (name, email, _parse_git_date(int(unix), tz)) + + +def _parse_git_date(unix: int, tz: str) -> datetime: + sign = 1 if tz[0] == "+" else -1 + hours = int(tz[1:3]) + minutes = int(tz[3:5]) + offset = timedelta(hours=hours, minutes=minutes) * sign + return datetime.fromtimestamp(unix, tz=timezone(offset)) + + +def _pack_bytes(repo_dir: Path) -> int: + pack_dir = repo_dir / "objects" / "pack" + if not pack_dir.is_dir(): + return 0 + return sum( + path.stat().st_size for path in pack_dir.glob("*.pack") if path.is_file() + ) + + +def _object_type_counts(repo_dir: Path) -> dict[str, int]: + output = _git( + repo_dir, + "cat-file", + "--batch-check=%(objecttype)", + "--batch-all-objects", + ).decode() + counts: dict[str, int] = {} + for line in output.splitlines(): + object_type = line.strip() + if object_type: + counts[object_type] = counts.get(object_type, 0) + 1 + return counts + + +def _git_executable() -> str: + git = shutil.which("git") + if git is None: + raise FetchFailedError("git executable not found") + return git + + +def _git(repo_dir: Path, *args: str, timeout: int = 30) -> bytes: + env = os.environ.copy() + env.update(_GIT_ENV) + command = [_git_executable(), "-C", str(repo_dir), *args] + try: + result = subprocess.run( # noqa: S603 + command, + check=False, + capture_output=True, + timeout=timeout, + env=env, + ) + except subprocess.TimeoutExpired as exc: + raise FetchFailedError(f"git {' '.join(args)} timed out") from exc + + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise FetchFailedError(f"git {' '.join(args)} failed: {stderr}") + return result.stdout diff --git a/backend/kernelCI_app/tests/unitTests/helpers/fixtures/git_commit_data.py b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/git_commit_data.py new file mode 100644 index 000000000..1822d7598 --- /dev/null +++ b/backend/kernelCI_app/tests/unitTests/helpers/fixtures/git_commit_data.py @@ -0,0 +1,20 @@ +MERGE_COMMIT_HASH = "0123456789abcdef0123456789abcdef01234567" +FIRST_PARENT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +SECOND_PARENT = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +# gpgsig uses space-prefixed continuation lines, including a "blank" signature line. +MERGE_COMMIT_OBJECT = ( + "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + "parent aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + "parent bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + "author Alice Author 1000000000 +0000\n" + "committer Bob Committer 1000000060 -0500\n" + "gpgsig -----BEGIN PGP SIGNATURE-----\n" + " \n" + " iQIzBAABCAAdFiEE\n" + " -----END PGP SIGNATURE-----\n" + "\n" + "Add feature foo\n" + "\n" + "Longer body that is not the subject.\n" +) diff --git a/backend/kernelCI_app/tests/unitTests/helpers/gitCommit_test.py b/backend/kernelCI_app/tests/unitTests/helpers/gitCommit_test.py new file mode 100644 index 000000000..36c2382e8 --- /dev/null +++ b/backend/kernelCI_app/tests/unitTests/helpers/gitCommit_test.py @@ -0,0 +1,248 @@ +import os +import shutil +import subprocess +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +from django.test import override_settings + +from kernelCI_app.helpers.gitCommit import ( + MAX_EPHEMERAL_PACK_BYTES, + CommitParseError, + FetchFailedError, + InvalidGitUrlError, + MissingGitUrlError, + OversizedPackError, + UnexpectedFetchObjectsError, + assert_single_commit_fetch, + fetch_commit_metadata, + parse_commit, + parse_commit_object, + resolve_checkout_git_url, + sanitize_git_url, +) +from kernelCI_app.tests.unitTests.helpers.fixtures.git_commit_data import ( + FIRST_PARENT, + MERGE_COMMIT_HASH, + MERGE_COMMIT_OBJECT, + SECOND_PARENT, +) + + +def _run_git(repo: Path, *args: str) -> str: + env = { + **os.environ, + "GIT_AUTHOR_NAME": "Alice Author", + "GIT_AUTHOR_EMAIL": "alice@example.com", + "GIT_COMMITTER_NAME": "Bob Committer", + "GIT_COMMITTER_EMAIL": "bob@example.com", + "GIT_AUTHOR_DATE": "2001-09-09T01:46:40+0000", + "GIT_COMMITTER_DATE": "2001-09-09T01:47:40-0500", + } + git = shutil.which("git") + assert git is not None + result = subprocess.run( # noqa: S603 + [git, "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env=env, + ) + return result.stdout.strip() + + +def _build_repo_with_merge(tmp_path: Path) -> tuple[Path, str, str, str]: + repo = tmp_path / "src" + repo.mkdir() + _run_git(repo, "init", "-b", "main") + _run_git(repo, "config", "user.name", "Alice Author") + _run_git(repo, "config", "user.email", "alice@example.com") + _run_git(repo, "config", "uploadpack.allowFilter", "true") + _run_git(repo, "config", "uploadpack.allowAnySHA1InWant", "true") + (repo / "a.txt").write_text("a\n") + _run_git(repo, "add", "a.txt") + _run_git(repo, "commit", "-m", "root commit") + root = _run_git(repo, "rev-parse", "HEAD") + + _run_git(repo, "checkout", "-b", "other") + (repo / "b.txt").write_text("b\n") + _run_git(repo, "add", "b.txt") + _run_git(repo, "commit", "-m", "side commit") + side = _run_git(repo, "rev-parse", "HEAD") + + _run_git(repo, "checkout", "main") + _run_git(repo, "merge", "--no-ff", "-m", "merge side\n\nmerge body\n", "other") + merge = _run_git(repo, "rev-parse", "HEAD") + return repo, root, side, merge + + +class TestParseCommitObject: + def test_author_committer_subject_ordered_parents(self): + metadata = parse_commit_object( + raw=MERGE_COMMIT_OBJECT, git_commit_hash=MERGE_COMMIT_HASH + ) + + assert metadata.git_commit_hash == MERGE_COMMIT_HASH + assert metadata.author_name == "Alice Author" + assert metadata.author_email == "alice@example.com" + assert metadata.author_date == datetime( + 2001, 9, 9, 1, 46, 40, tzinfo=timezone.utc + ) + assert metadata.committer_name == "Bob Committer" + assert metadata.committer_email == "bob@example.com" + assert metadata.committer_date == datetime( + 2001, 9, 8, 20, 47, 40, tzinfo=timezone(timedelta(hours=-5)) + ) + assert metadata.subject == "Add feature foo" + assert metadata.message.startswith("Add feature foo\n") + assert "Longer body" in metadata.message + assert metadata.parent_hashes == (FIRST_PARENT, SECOND_PARENT) + + def test_missing_separator_raises(self): + with pytest.raises(CommitParseError): + parse_commit_object(raw="tree abc\n", git_commit_hash=MERGE_COMMIT_HASH) + + +class TestParseCommitFromRepo: + def test_parse_existing_repo_does_not_fetch(self, tmp_path, monkeypatch): + repo, root, side, merge = _build_repo_with_merge(tmp_path) + calls: list[list[str]] = [] + real_run = subprocess.run + + def wrapped(*args, **kwargs): + command = args[0] if args else kwargs.get("args") + if isinstance(command, list): + calls.append(command) + return real_run(*args, **kwargs) + + monkeypatch.setattr("kernelCI_app.helpers.gitCommit.subprocess.run", wrapped) + + metadata = parse_commit(repo_path=str(repo), git_commit_hash=merge) + + assert metadata.git_commit_hash == merge + assert metadata.parent_hashes == (root, side) + assert metadata.subject == "merge side" + assert metadata.message.startswith("merge side\n") + assert all("fetch" not in command for command in calls) + + +class TestSanitizeGitUrl: + def test_accepts_https_and_file(self): + https = "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/" + assert ( + sanitize_git_url(https) + == "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + ) + assert sanitize_git_url("file:///tmp/linux.git") == "file:///tmp/linux.git" + + def test_malformed_does_not_raise(self): + for url in ( + "", + " ", + None, + "https://example.com/", + "https://example.com", + "linux.git", + "git@github.com:org/repo.git", + "not a url at all!!!", + ): + assert sanitize_git_url(url) is None + + +class TestResolveCheckoutGitUrl: + @patch("kernelCI_app.helpers.gitCommit.Checkouts.objects") + def test_prefers_maestro_kernel_org(self, mock_objects): + rows = mock_objects.filter.return_value.values_list.return_value + rows.distinct.return_value = [ + ("redhat", "https://github.com/foo/linux.git"), + ( + "maestro", + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git", + ), + ("maestro", "https://github.com/torvalds/linux.git"), + ] + assert resolve_checkout_git_url("abc") == ( + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + ) + + @patch("kernelCI_app.helpers.gitCommit.Checkouts.objects") + def test_skips_malformed_urls(self, mock_objects): + rows = mock_objects.filter.return_value.values_list.return_value + rows.distinct.return_value = [ + ("maestro", "git@github.com:org/repo.git"), + ("maestro", "https://example.com/"), + ( + "redhat", + "https://git.kernel.org/pub/scm/linux/kernel/git/redhat/linux.git", + ), + ] + assert resolve_checkout_git_url("abc") == ( + "https://git.kernel.org/pub/scm/linux/kernel/git/redhat/linux.git" + ) + + +class TestFetchCommitMetadata: + def test_fetch_from_local_repo_then_wipes(self, tmp_path): + repo, _root, _side, merge = _build_repo_with_merge(tmp_path) + scratch = tmp_path / "scratch" + with override_settings(GIT_SCRATCH_DIR=str(scratch)): + metadata = fetch_commit_metadata(merge, url=f"file://{repo}") + + assert metadata.git_commit_hash == merge + assert metadata.subject == "merge side" + assert metadata.parent_hashes[0] == _root + leftover = list(scratch.glob("commit-fetch-*")) + assert leftover == [] + + def test_fetch_failure(self, tmp_path): + scratch = tmp_path / "scratch" + with override_settings(GIT_SCRATCH_DIR=str(scratch)): + with pytest.raises(FetchFailedError): + fetch_commit_metadata("a" * 40, url="file:///no/such/repo.git") + leftover = list(scratch.glob("commit-fetch-*")) + assert leftover == [] + + def test_bad_url(self): + with pytest.raises(InvalidGitUrlError): + fetch_commit_metadata("a" * 40, url="git@github.com:org/repo.git") + + @patch("kernelCI_app.helpers.gitCommit.resolve_checkout_git_url", return_value=None) + def test_missing_url(self, _mock_resolve): + with pytest.raises(MissingGitUrlError): + fetch_commit_metadata("a" * 40) + + def test_oversized_pack(self, tmp_path): + repo = tmp_path / "bare.git" + (repo / "objects" / "pack").mkdir(parents=True) + pack = repo / "objects" / "pack" / "pack-deadbeef.pack" + pack.write_bytes(b"\0" * (MAX_EPHEMERAL_PACK_BYTES + 1)) + with pytest.raises(OversizedPackError): + assert_single_commit_fetch(repo) + + def test_full_pack_extra_commits(self, tmp_path, monkeypatch): + repo = tmp_path / "bare.git" + repo.mkdir() + monkeypatch.setattr( + "kernelCI_app.helpers.gitCommit._object_type_counts", + lambda _repo: {"commit": 12}, + ) + monkeypatch.setattr( + "kernelCI_app.helpers.gitCommit._pack_bytes", lambda _repo: 10 + ) + with pytest.raises(UnexpectedFetchObjectsError): + assert_single_commit_fetch(repo) + + def test_full_pack_includes_trees(self, tmp_path, monkeypatch): + repo = tmp_path / "bare.git" + repo.mkdir() + monkeypatch.setattr( + "kernelCI_app.helpers.gitCommit._object_type_counts", + lambda _repo: {"commit": 1, "tree": 4, "blob": 20}, + ) + monkeypatch.setattr( + "kernelCI_app.helpers.gitCommit._pack_bytes", lambda _repo: 10 + ) + with pytest.raises(UnexpectedFetchObjectsError): + assert_single_commit_fetch(repo)