From fa5f46d5b93f87ce629d643e754901d950db505b Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Tue, 27 Oct 2026 11:03:02 +0100 Subject: [PATCH] contest: remote: fetcher: retry on git fetch errors Recently, a bunch of services failed, because 'git fetch' failed. It looks like the CI got rate-limited, but there are no clear limits on Git operations apparently: it depends on other traffic usage. So let's retry, with a limited exponential backoff, waiting max ~32.5 minutes. Hopefully it would work instead of crashing. Signed-off-by: Matthieu Baerts --- contest/remote/lib/fetcher.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/contest/remote/lib/fetcher.py b/contest/remote/lib/fetcher.py index e8cf472..ce144e3 100644 --- a/contest/remote/lib/fetcher.py +++ b/contest/remote/lib/fetcher.py @@ -136,6 +136,22 @@ def _find_branch(self, name): print("Unexpected number of branches found:", branches) return branches[0] + def _git_fetch_all(self, n): + last = n - 1 + sleep = 30 + + for i in range(n): + try: + subprocess.run('git fetch --all --prune', cwd=self._tree_path, + shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Fetching failed (attempt {i + 1})", repr(e)) + if i == last: + raise + # with n = 10, max 32.5 minutes + time.sleep(min(sleep, 300)) + sleep *= 2 + def _run_once(self): try: r = self._http.request('GET', self._branches_url) @@ -165,8 +181,7 @@ def _run_once(self): shell=True) # For now assume URL is in one of the remotes - subprocess.run('git fetch --all --prune', cwd=self._tree_path, - shell=True, check=True) + self._git_fetch_all() # After upgrading git 2.40.1 -> 2.47.1 CI hits a race in git, # where tree is locked, even though previous command has finished.