From cde49bc22187aadf784ba8b1bb52ddc10484d718 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 14:40:30 +0100 Subject: [PATCH 01/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index e1ef3da..349f5d9 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -192,11 +192,10 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: - if "Host key verification failed." in e.stderr or "Permission denied (publickey)" in e.stderr: - try: - git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) - except Exception as e_inner: - raise e_inner + try: + git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) + except Exception as e_inner: + raise e_inner else: raise e instance = cls(to_path) From 55f019bdaa3c4d663949753de569f045b9e35af6 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 14:41:42 +0100 Subject: [PATCH 02/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 349f5d9..7cb9ff8 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -192,6 +192,8 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: + print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)}") + traceback.print_exc() try: git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: From 2269742e8189146afb28660160ed8c2581abebf6 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 14:56:06 +0100 Subject: [PATCH 03/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 7cb9ff8..3a6e788 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -187,7 +187,16 @@ def update(self): @classmethod def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **kwargs): # prevent git terminal prompts from interrupting the process. + previous_git_terminal_prompt = None + if "GIT_TERMINAL_PROMPT" in os.environ: + previous_git_terminal_prompt = os.environ["GIT_TERMINAL_PROMPT"] + + previous_git_ssh_command = None + if "GIT_SSH_COMMAND" in os.environ: + previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] + os.environ["GIT_TERMINAL_PROMPT"] = "0" + os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) @@ -198,8 +207,16 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: raise e_inner + finally: + if previous_git_terminal_prompt is None: + os.environ.pop("GIT_TERMINAL_PROMPT") else: - raise e + os.environ["GIT_TERMINAL_PROMPT"] = previous_git_terminal_prompt + if previous_git_ssh_command is None: + os.environ.pop("GIT_SSH_COMMAND") + else: + os.environ["GIT_SSH_COMMAND"] = previous_git_ssh_command + instance = cls(to_path) return instance From 978bf3d3c96708a17bd69bda441c1831aaa89f6c Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:01:44 +0100 Subject: [PATCH 04/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 3a6e788..de481f9 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -195,9 +195,11 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k if "GIT_SSH_COMMAND" in os.environ: previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] - os.environ["GIT_TERMINAL_PROMPT"] = "0" + # os.environ["GIT_TERMINAL_PROMPT"] = "0" os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" + print("Cloning", url) + try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: From e704cfa980c421d7de764af66169d19f71caaabc Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:04:20 +0100 Subject: [PATCH 05/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index de481f9..19e720e 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -195,7 +195,7 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k if "GIT_SSH_COMMAND" in os.environ: previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] - # os.environ["GIT_TERMINAL_PROMPT"] = "0" + os.environ["GIT_TERMINAL_PROMPT"] = "0" os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" print("Cloning", url) From e5a36bece8b3543c35c4f5ad25a98765c0c86381 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:07:45 +0100 Subject: [PATCH 06/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 46 +++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 19e720e..85adf1a 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -187,41 +187,47 @@ def update(self): @classmethod def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **kwargs): # prevent git terminal prompts from interrupting the process. - previous_git_terminal_prompt = None - if "GIT_TERMINAL_PROMPT" in os.environ: - previous_git_terminal_prompt = os.environ["GIT_TERMINAL_PROMPT"] - - previous_git_ssh_command = None - if "GIT_SSH_COMMAND" in os.environ: - previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] - - os.environ["GIT_TERMINAL_PROMPT"] = "0" - os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" + previous_git_ssh_command, previous_git_terminal_prompt = cls.git_environ_setup() print("Cloning", url) try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: - print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)}") - traceback.print_exc() + print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)} with {e}") try: git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: raise e_inner finally: - if previous_git_terminal_prompt is None: - os.environ.pop("GIT_TERMINAL_PROMPT") - else: - os.environ["GIT_TERMINAL_PROMPT"] = previous_git_terminal_prompt - if previous_git_ssh_command is None: - os.environ.pop("GIT_SSH_COMMAND") - else: - os.environ["GIT_SSH_COMMAND"] = previous_git_ssh_command + cls.git_environ_reset(previous_git_ssh_command, previous_git_terminal_prompt) instance = cls(to_path) return instance + @classmethod + def git_environ_setup(cls): + previous_git_terminal_prompt = None + if "GIT_TERMINAL_PROMPT" in os.environ: + previous_git_terminal_prompt = os.environ["GIT_TERMINAL_PROMPT"] + previous_git_ssh_command = None + if "GIT_SSH_COMMAND" in os.environ: + previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] + os.environ["GIT_TERMINAL_PROMPT"] = "0" + os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" + return previous_git_ssh_command, previous_git_terminal_prompt + + @classmethod + def git_environ_reset(cls, previous_git_ssh_command, previous_git_terminal_prompt): + if previous_git_terminal_prompt is None: + os.environ.pop("GIT_TERMINAL_PROMPT") + else: + os.environ["GIT_TERMINAL_PROMPT"] = previous_git_terminal_prompt + if previous_git_ssh_command is None: + os.environ.pop("GIT_SSH_COMMAND") + else: + os.environ["GIT_SSH_COMMAND"] = previous_git_ssh_command + def add_remote(self, remote_url, remote_name=None): """ Add a remote to the repository. From 34c1433c67cecfbce289fc65eae8b47b9015ef55 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:09:06 +0100 Subject: [PATCH 07/12] Add fallback to pull that also tries the HTTP URL --- cadetrdm/repositories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 85adf1a..df906e2 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -194,7 +194,7 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: - print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)} with {e}") + print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)} with {e, e.stderr, e.stdout}") try: git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: From 9c06444658aab19a1e65213bc7fcb3bf0d0e878f Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:18:21 +0100 Subject: [PATCH 08/12] Add fallback --- cadetrdm/repositories.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index df906e2..757ff68 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -187,46 +187,48 @@ def update(self): @classmethod def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **kwargs): # prevent git terminal prompts from interrupting the process. - previous_git_ssh_command, previous_git_terminal_prompt = cls.git_environ_setup() + previous_environment_variables = cls._git_environ_setup() print("Cloning", url) try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: - print(f"Clone from {url} failed. Trying clone from {ssh_url_to_http_url(url)} with {e, e.stderr, e.stdout}") try: git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: + print(f"Clone from {url} failed with {e, e.stderr, e.stdout}") + print(f"and clone from {ssh_url_to_http_url(url)} failed with: ") raise e_inner + finally: + cls._git_environ_reset(previous_environment_variables) finally: - cls.git_environ_reset(previous_git_ssh_command, previous_git_terminal_prompt) + cls._git_environ_reset(previous_environment_variables) instance = cls(to_path) return instance - @classmethod - def git_environ_setup(cls): - previous_git_terminal_prompt = None + @staticmethod + def _git_environ_setup(): + environment_variables = { + "GIT_TERMINAL_PROMPT": None, + "GIT_SSH_COMMAND": None + } if "GIT_TERMINAL_PROMPT" in os.environ: - previous_git_terminal_prompt = os.environ["GIT_TERMINAL_PROMPT"] - previous_git_ssh_command = None + environment_variables["GIT_TERMINAL_PROMPT"] = os.environ["GIT_TERMINAL_PROMPT"] if "GIT_SSH_COMMAND" in os.environ: - previous_git_ssh_command = os.environ["GIT_SSH_COMMAND"] + environment_variables["GIT_SSH_COMMAND"] = os.environ["GIT_SSH_COMMAND"] os.environ["GIT_TERMINAL_PROMPT"] = "0" os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" - return previous_git_ssh_command, previous_git_terminal_prompt + return environment_variables - @classmethod - def git_environ_reset(cls, previous_git_ssh_command, previous_git_terminal_prompt): - if previous_git_terminal_prompt is None: - os.environ.pop("GIT_TERMINAL_PROMPT") - else: - os.environ["GIT_TERMINAL_PROMPT"] = previous_git_terminal_prompt - if previous_git_ssh_command is None: - os.environ.pop("GIT_SSH_COMMAND") - else: - os.environ["GIT_SSH_COMMAND"] = previous_git_ssh_command + @staticmethod + def _git_environ_reset(previous_environment_variables): + for key, previous_value in previous_environment_variables.items(): + if previous_value is None: + os.environ.pop(key) + else: + os.environ[key] = previous_value def add_remote(self, remote_url, remote_name=None): """ From b7ca4bd61cc7168d6ff9ba2ccd5175a2edbd95b3 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:19:57 +0100 Subject: [PATCH 09/12] Add fallback --- cadetrdm/repositories.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index 757ff68..a723bc5 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -200,8 +200,6 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k print(f"Clone from {url} failed with {e, e.stderr, e.stdout}") print(f"and clone from {ssh_url_to_http_url(url)} failed with: ") raise e_inner - finally: - cls._git_environ_reset(previous_environment_variables) finally: cls._git_environ_reset(previous_environment_variables) From 291cb4016915afd29cf4f12bc00877d9edb7040c Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:42:48 +0100 Subject: [PATCH 10/12] Add fallback --- cadetrdm/repositories.py | 33 +++++++++++++++------------------ tests/test_git_adapter.py | 30 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index a723bc5..ebd618c 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -189,8 +189,6 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k # prevent git terminal prompts from interrupting the process. previous_environment_variables = cls._git_environ_setup() - print("Cloning", url) - try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: @@ -209,16 +207,17 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k @staticmethod def _git_environ_setup(): environment_variables = { - "GIT_TERMINAL_PROMPT": None, - "GIT_SSH_COMMAND": None + "GIT_TERMINAL_PROMPT": "0", + "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=yes", + "GIT_CLONE_PROTECTION_ACTIVE": "0", } - if "GIT_TERMINAL_PROMPT" in os.environ: - environment_variables["GIT_TERMINAL_PROMPT"] = os.environ["GIT_TERMINAL_PROMPT"] - if "GIT_SSH_COMMAND" in os.environ: - environment_variables["GIT_SSH_COMMAND"] = os.environ["GIT_SSH_COMMAND"] - os.environ["GIT_TERMINAL_PROMPT"] = "0" - os.environ["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=yes" - return environment_variables + previous_environment_variables = {key: None for key in environment_variables.keys()} + for key, value in environment_variables.items(): + if key in os.environ: + previous_environment_variables[key] = os.environ[key] + os.environ[key] = value + + return previous_environment_variables @staticmethod def _git_environ_reset(previous_environment_variables): @@ -788,17 +787,15 @@ def _clone_output_repo(self, multi_options: List[str] = None): output_remotes = metadata["output_remotes"] output_path = self.path / output_remotes["output_folder_name"] ssh_remotes = list(output_remotes["output_remotes"].values()) - http_remotes = [ssh_url_to_http_url(url) for url in ssh_remotes] - if len(ssh_remotes + http_remotes) == 0: + if len(ssh_remotes) == 0: warnings.warn("No output remotes configured in .cadet-rdm-data.json") - for output_remote in ssh_remotes + http_remotes: + for output_remote in ssh_remotes: try: print(f"Attempting to clone {output_remote} into {output_path}") - _ = self.clone_from(output_remote, output_path, multi_options=multi_options) - except Exception as e: - print(e) - else: + _ = OutputRepo.clone_from(output_remote, output_path, multi_options=multi_options) break + except Exception: + traceback.print_exc() @staticmethod def _add_jupytext_file(path_root: str | Path = "."): diff --git a/tests/test_git_adapter.py b/tests/test_git_adapter.py index 813dbe1..5daed9b 100644 --- a/tests/test_git_adapter.py +++ b/tests/test_git_adapter.py @@ -277,23 +277,23 @@ def test_error_stack(): def test_cadet_rdm(path_to_repo): # because these depend on one-another and there is no native support afaik for sequential tests # these tests are called sequentially here as try_ functions. - try_initialize_git_repo(path_to_repo) + # try_initialize_git_repo(path_to_repo) try_initialize_from_remote() - try_add_remote(path_to_repo) - # try_add_submodule(path_to_repo) - try_commit_code(path_to_repo) - try_commit_code_without_code_changes(path_to_repo) - try_commit_results_with_uncommitted_code_changes(path_to_repo) - try_output_function(path_to_repo) - - try_commit_results_with_options(path_to_repo) - results_branch_name = try_commit_results_data(path_to_repo) - try_print_log(path_to_repo) - - try_commit_code(path_to_repo) - - try_load_previous_output(path_to_repo, results_branch_name) + # try_add_remote(path_to_repo) + # # try_add_submodule(path_to_repo) + # try_commit_code(path_to_repo) + # try_commit_code_without_code_changes(path_to_repo) + # try_commit_results_with_uncommitted_code_changes(path_to_repo) + # try_output_function(path_to_repo) + # + # try_commit_results_with_options(path_to_repo) + # results_branch_name = try_commit_results_data(path_to_repo) + # try_print_log(path_to_repo) + # + # try_commit_code(path_to_repo) + # + # try_load_previous_output(path_to_repo, results_branch_name) def test_with_detached_head(): From 66e9689a248f06807d3dd35c5816a744b7254f15 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:51:47 +0100 Subject: [PATCH 11/12] Add fallback --- cadetrdm/repositories.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cadetrdm/repositories.py b/cadetrdm/repositories.py index ebd618c..68539be 100644 --- a/cadetrdm/repositories.py +++ b/cadetrdm/repositories.py @@ -192,11 +192,12 @@ def clone_from(cls, url, to_path, multi_options: Optional[List[str]] = None, **k try: git.Repo.clone_from(url, to_path, multi_options=multi_options, **kwargs) except git.exc.GitCommandError as e: + print(f"Clone from {url} failed with {e, e.stderr, e.stdout}.") + print(f"Retrying with {ssh_url_to_http_url(url)}.") try: git.Repo.clone_from(ssh_url_to_http_url(url), to_path, multi_options=multi_options, **kwargs) except Exception as e_inner: - print(f"Clone from {url} failed with {e, e.stderr, e.stdout}") - print(f"and clone from {ssh_url_to_http_url(url)} failed with: ") + print(f"Clone from {ssh_url_to_http_url(url)} failed with: ") raise e_inner finally: cls._git_environ_reset(previous_environment_variables) From eeb6026306c5a85a568e7297835b9604fb6179f2 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Tue, 11 Feb 2025 15:52:07 +0100 Subject: [PATCH 12/12] Add fallback --- tests/test_git_adapter.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_git_adapter.py b/tests/test_git_adapter.py index 5daed9b..813dbe1 100644 --- a/tests/test_git_adapter.py +++ b/tests/test_git_adapter.py @@ -277,23 +277,23 @@ def test_error_stack(): def test_cadet_rdm(path_to_repo): # because these depend on one-another and there is no native support afaik for sequential tests # these tests are called sequentially here as try_ functions. - # try_initialize_git_repo(path_to_repo) + try_initialize_git_repo(path_to_repo) try_initialize_from_remote() - # try_add_remote(path_to_repo) - # # try_add_submodule(path_to_repo) - # try_commit_code(path_to_repo) - # try_commit_code_without_code_changes(path_to_repo) - # try_commit_results_with_uncommitted_code_changes(path_to_repo) - # try_output_function(path_to_repo) - # - # try_commit_results_with_options(path_to_repo) - # results_branch_name = try_commit_results_data(path_to_repo) - # try_print_log(path_to_repo) - # - # try_commit_code(path_to_repo) - # - # try_load_previous_output(path_to_repo, results_branch_name) + try_add_remote(path_to_repo) + # try_add_submodule(path_to_repo) + try_commit_code(path_to_repo) + try_commit_code_without_code_changes(path_to_repo) + try_commit_results_with_uncommitted_code_changes(path_to_repo) + try_output_function(path_to_repo) + + try_commit_results_with_options(path_to_repo) + results_branch_name = try_commit_results_data(path_to_repo) + try_print_log(path_to_repo) + + try_commit_code(path_to_repo) + + try_load_previous_output(path_to_repo, results_branch_name) def test_with_detached_head():