Skip to content

Make Git mirror recovery safe #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
programmersd21 merged 27 commits into from
Oct 10, 2026
Merged

Make Git mirror recovery safe #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
d135aa8
Remove partial Git mirror after clone failure
programmersd21 Oct 10, 2026
af0eaaa
Remove partial Git mirror after clone failure
programmersd21 Oct 10, 2026
2e3ec86
Test recovery from an incomplete Git mirror cache
programmersd21 Oct 10, 2026
018b729
Make mirror recovery regression test portable and retry-aware
programmersd21 Oct 10, 2026
1c7c58a
Target the installed package mirror in recovery test
programmersd21 Oct 10, 2026
5525fa4
Select the mirror using its URL-derived cache slug
programmersd21 Oct 10, 2026
2fb9081
Simulate a genuinely incomplete bare Git mirror
programmersd21 Oct 10, 2026
bb81f33
Retry Git mirror clone after cleaning partial cache
programmersd21 Oct 10, 2026
54ce549
Retry Git mirror clone after cleaning partial cache
programmersd21 Oct 10, 2026
f6b8fe0
Assert package install repairs an incomplete mirror in one run
programmersd21 Oct 10, 2026
b35b96f
Clean partial Git mirror after clone failure
programmersd21 Oct 10, 2026
a8df580
Clean partial Git mirror after clone failure
programmersd21 Oct 10, 2026
5f17a0f
Verify an incomplete mirror is cleaned and retry succeeds
programmersd21 Oct 10, 2026
ba3a1e0
Make mirror repair symlink-safe and serialize concurrent installs
programmersd21 Oct 10, 2026
f6d2a77
Make mirror repair symlink-safe and serialize concurrent installs
programmersd21 Oct 10, 2026
665ad27
Test single-run mirror recovery and symlink safety
programmersd21 Oct 10, 2026
1f61169
Cover concurrent mirror initialization
programmersd21 Oct 10, 2026
9abae29
Reject symlinked mirror paths before reuse
programmersd21 Oct 10, 2026
910f1a0
Reject symlinked mirror paths before reuse
programmersd21 Oct 10, 2026
5be5eb0
Format mirror symlink check
programmersd21 Oct 10, 2026
b329515
Format mirror symlink check
programmersd21 Oct 10, 2026
1b27396
Fix GCC format-truncation warnings in mirror errors
programmersd21 Oct 10, 2026
46801b1
Fix GCC format-truncation warnings in mirror errors
programmersd21 Oct 10, 2026
5e346b5
Keep Windows builds warning-clean
programmersd21 Oct 10, 2026
c8d2acb
Keep Windows builds warning-clean
programmersd21 Oct 10, 2026
9d81e6f
Guard mirror recovery test against an empty path
programmersd21 Oct 10, 2026
b1f6743
Merge branch 'main' into fix/git-mirror-cache-recovery
programmersd21 Oct 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 104 additions & 7 deletions rust/flintlang/c-src/src/pkg.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
Expand Down Expand Up @@ -837,7 +838,7 @@ static bool pkg_satisfies(const char *requirement,
* ours saying which URL it was for -- git already explains itself, and
* wrapping that twice helps nobody.
*/
static bool pkg_ensure_mirror(const char *mirror,
static bool pkg_ensure_mirror_unlocked(const char *mirror,
const char *url,
bool fetch,
const char *commit,
Expand All @@ -849,21 +850,69 @@ static bool pkg_ensure_mirror(const char *mirror,
if (snprintf(head, sizeof(head), "%s/HEAD", mirror) >=
(int)sizeof(head)) {
snprintf(error, error_size, "path too long");
return NULL;
return false;
}
if (access(head, F_OK) != 0) {
struct stat mirror_stat;
bool mirror_is_link = lstat(mirror, &mirror_stat) == 0 &&
S_ISLNK(mirror_stat.st_mode);
if (mirror_is_link || access(head, F_OK) != 0) {
char staging[4352];
bool made_staging = false;
for (unsigned int attempt = 0; attempt < 100; attempt++) {
int n = snprintf(staging,
sizeof(staging),
"%s.tmp.%ld.%u",
mirror,
(long)getpid(),
attempt);
if (n < 0 || (size_t)n >= sizeof(staging)) {
snprintf(error, error_size, "path too long");
return false;
}
if (mkdir(staging, 0700) == 0) {
made_staging = true;
break;
}
if (errno != EEXIST) {
snprintf(error, error_size, "cannot create mirror staging directory");
return false;
}
}
if (!made_staging) {
snprintf(error, error_size, "cannot create mirror staging directory");
return false;
}
char *argv[] = {"git",
"-c",
"protocol.file.allow=always",
"clone",
"--mirror",
"--",
(char *)url,
(char *)mirror,
staging,
NULL};
if (pkg_git(argv, NULL, 1) != 0) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot clone '%s'", url);
return NULL;
return false;
}

struct stat st;
if (lstat(mirror, &st) == 0) {
if (!pkg_remove_tree(mirror)) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot remove damaged mirror");
return false;
}
} else if (errno != ENOENT) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot inspect mirror");
return false;
}
if (rename(staging, mirror) != 0) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot publish mirror");
return false;
}
return true;
}
Expand All @@ -884,7 +933,7 @@ static bool pkg_ensure_mirror(const char *mirror,
NULL};
if (pkg_git(argv, NULL, 1) != 0) {
snprintf(error, error_size, "cannot fetch '%s'", url);
return NULL;
return false;
}
}
return true;
Expand All @@ -898,6 +947,47 @@ static bool pkg_ensure_mirror(const char *mirror,
#endif
}

static bool pkg_ensure_mirror(const char *mirror,
const char *url,
bool fetch,
const char *commit,
char *error,
size_t error_size)
{
#ifndef _WIN32
char lock_path[4352];
if (snprintf(lock_path, sizeof(lock_path), "%s.lock", mirror) >=
(int)sizeof(lock_path)) {
snprintf(error, error_size, "path too long");
return false;
}
int lock_fd = open(lock_path, O_CREAT | O_RDWR, 0600);
if (lock_fd < 0) {
snprintf(error, error_size, "cannot open mirror lock");
return false;
}
struct flock lock = {0};
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
while (fcntl(lock_fd, F_SETLKW, &lock) != 0) {
if (errno == EINTR)
continue;
close(lock_fd);
snprintf(error, error_size, "cannot lock mirror");
return false;
}
bool ok = pkg_ensure_mirror_unlocked(
mirror, url, fetch, commit, error, error_size);
lock.l_type = F_UNLCK;
(void)fcntl(lock_fd, F_SETLK, &lock);
close(lock_fd);
return ok;
#else
return pkg_ensure_mirror_unlocked(
mirror, url, fetch, commit, error, error_size);
#endif
}

/* copy a commit out of a mirror into dest: clone, detach at the pin,
* drop the .git, so what lands in flint_modules/ is source, not a
* repository. dest must not exist; the caller clears it first. */
Expand Down Expand Up @@ -1124,9 +1214,16 @@ static bool pkg_remove_entry(const char *child)

static bool pkg_remove_tree(const char *path)
{
struct stat st;
int sr = pkg_stat_entry(path, &st);
if (sr < 0)
return false;
if (sr != 0 || !S_ISDIR(st.st_mode))
return remove(path) == 0;

DIR *dir = opendir(path);
if (dir == NULL)
return remove(path) == 0;
return false;
bool ok = true;
struct dirent *entry;
while (ok && (entry = readdir(dir)) != NULL) {
Expand Down
111 changes: 104 additions & 7 deletions src/pkg.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
Expand Down Expand Up @@ -837,7 +838,7 @@ static bool pkg_satisfies(const char *requirement,
* ours saying which URL it was for -- git already explains itself, and
* wrapping that twice helps nobody.
*/
static bool pkg_ensure_mirror(const char *mirror,
static bool pkg_ensure_mirror_unlocked(const char *mirror,
const char *url,
bool fetch,
const char *commit,
Expand All @@ -849,21 +850,69 @@ static bool pkg_ensure_mirror(const char *mirror,
if (snprintf(head, sizeof(head), "%s/HEAD", mirror) >=
(int)sizeof(head)) {
snprintf(error, error_size, "path too long");
return NULL;
return false;
}
if (access(head, F_OK) != 0) {
struct stat mirror_stat;
bool mirror_is_link = lstat(mirror, &mirror_stat) == 0 &&
S_ISLNK(mirror_stat.st_mode);
if (mirror_is_link || access(head, F_OK) != 0) {
char staging[4352];
bool made_staging = false;
for (unsigned int attempt = 0; attempt < 100; attempt++) {
int n = snprintf(staging,
sizeof(staging),
"%s.tmp.%ld.%u",
mirror,
(long)getpid(),
attempt);
if (n < 0 || (size_t)n >= sizeof(staging)) {
snprintf(error, error_size, "path too long");
return false;
}
if (mkdir(staging, 0700) == 0) {
made_staging = true;
break;
}
if (errno != EEXIST) {
snprintf(error, error_size, "cannot create mirror staging directory");
return false;
}
}
if (!made_staging) {
snprintf(error, error_size, "cannot create mirror staging directory");
return false;
}
char *argv[] = {"git",
"-c",
"protocol.file.allow=always",
"clone",
"--mirror",
"--",
(char *)url,
(char *)mirror,
staging,
NULL};
if (pkg_git(argv, NULL, 1) != 0) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot clone '%s'", url);
return NULL;
return false;
}

struct stat st;
if (lstat(mirror, &st) == 0) {
if (!pkg_remove_tree(mirror)) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot remove damaged mirror");
return false;
}
} else if (errno != ENOENT) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot inspect mirror");
return false;
}
if (rename(staging, mirror) != 0) {
(void)pkg_remove_tree(staging);
snprintf(error, error_size, "cannot publish mirror");
return false;
}
return true;
}
Expand All @@ -884,7 +933,7 @@ static bool pkg_ensure_mirror(const char *mirror,
NULL};
if (pkg_git(argv, NULL, 1) != 0) {
snprintf(error, error_size, "cannot fetch '%s'", url);
return NULL;
return false;
}
}
return true;
Expand All @@ -898,6 +947,47 @@ static bool pkg_ensure_mirror(const char *mirror,
#endif
}

static bool pkg_ensure_mirror(const char *mirror,
const char *url,
bool fetch,
const char *commit,
char *error,
size_t error_size)
{
#ifndef _WIN32
char lock_path[4352];
if (snprintf(lock_path, sizeof(lock_path), "%s.lock", mirror) >=
(int)sizeof(lock_path)) {
snprintf(error, error_size, "path too long");
return false;
}
int lock_fd = open(lock_path, O_CREAT | O_RDWR, 0600);
if (lock_fd < 0) {
snprintf(error, error_size, "cannot open mirror lock");
return false;
}
struct flock lock = {0};
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
while (fcntl(lock_fd, F_SETLKW, &lock) != 0) {
if (errno == EINTR)
continue;
close(lock_fd);
snprintf(error, error_size, "cannot lock mirror");
return false;
}
bool ok = pkg_ensure_mirror_unlocked(
mirror, url, fetch, commit, error, error_size);
lock.l_type = F_UNLCK;
(void)fcntl(lock_fd, F_SETLK, &lock);
close(lock_fd);
return ok;
#else
return pkg_ensure_mirror_unlocked(
mirror, url, fetch, commit, error, error_size);
#endif
}

/* copy a commit out of a mirror into dest: clone, detach at the pin,
* drop the .git, so what lands in flint_modules/ is source, not a
* repository. dest must not exist; the caller clears it first. */
Expand Down Expand Up @@ -1124,9 +1214,16 @@ static bool pkg_remove_entry(const char *child)

static bool pkg_remove_tree(const char *path)
{
struct stat st;
int sr = pkg_stat_entry(path, &st);
if (sr < 0)
return false;
if (sr != 0 || !S_ISDIR(st.st_mode))
return remove(path) == 0;

DIR *dir = opendir(path);
if (dir == NULL)
return remove(path) == 0;
return false;
bool ok = true;
struct dirent *entry;
while (ok && (entry = readdir(dir)) != NULL) {
Expand Down
Loading