Handle GHES 404 errors when URLs have expired

dpmex4527 · web-flow · commit c8e241dec8ee · 2025-10-14T15:18:04.000Z
- Add tests
diff --git a/src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs
@@ -2033,10 +2033,270 @@ public async Task Both_Archives_Download_Retry_On_403_Error()
             _mockHttpDownloadService.Verify(x => x.DownloadToFile(freshMetadataArchiveUrl, METADATA_ARCHIVE_FILE_PATH), Times.Once);
         }
 
+        [Fact]
+        public async Task Git_Archive_Download_Retries_On_404_Error()
+        {
+            // Arrange
+            var freshGitArchiveUrl = "https://example.com/1/fresh";
+
+            _mockTargetGithubApi.Setup(x => x.GetOrganizationId(TARGET_ORG).Result).Returns(GITHUB_ORG_ID);
+            _mockTargetGithubApi.Setup(x => x.CreateGhecMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID);
+            _mockTargetGithubApi.Setup(x => x.DoesOrgExist(TARGET_ORG).Result).Returns(true);
+            _mockTargetGithubApi.Setup(x => x.StartMigration(
+                MIGRATION_SOURCE_ID,
+                GHES_REPO_URL,
+                GITHUB_ORG_ID,
+                TARGET_REPO,
+                GITHUB_SOURCE_PAT,
+                GITHUB_TARGET_PAT,
+                AUTHENTICATED_GIT_ARCHIVE_URL,
+                AUTHENTICATED_METADATA_ARCHIVE_URL,
+                false,
+                null,
+                false).Result).Returns(MIGRATION_ID);
+            _mockTargetGithubApi.Setup(x => x.GetMigration(MIGRATION_ID).Result).Returns((State: RepositoryMigrationStatus.Succeeded, TARGET_REPO, 0, null, null));
+
+            _mockSourceGithubApi.Setup(x => x.StartGitArchiveGeneration(SOURCE_ORG, SOURCE_REPO).Result).Returns(GIT_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.StartMetadataArchiveGeneration(SOURCE_ORG, SOURCE_REPO, false, false).Result).Returns(METADATA_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, GIT_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, METADATA_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+
+            // Setup GetArchiveMigrationUrl to return original URL first, then fresh URL on retry
+            _mockSourceGithubApi.SetupSequence(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID).Result)
+                .Returns(GIT_ARCHIVE_URL)
+                .Returns(freshGitArchiveUrl);
+
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID).Result).Returns(METADATA_ARCHIVE_URL);
+
+            // Setup HttpDownloadService to fail first time with 404, succeed on retry
+            _mockHttpDownloadService
+                .SetupSequence(x => x.DownloadToFile(It.IsAny<string>(), GIT_ARCHIVE_FILE_PATH))
+                .ThrowsAsync(CreateHttpNotFoundException())
+                .Returns(Task.CompletedTask);
+
+            _mockHttpDownloadService
+                .Setup(x => x.DownloadToFile(METADATA_ARCHIVE_URL, METADATA_ARCHIVE_FILE_PATH))
+                .Returns(Task.CompletedTask);
+
+            _mockAzureApi
+                .SetupSequence(x => x.UploadToBlob(It.IsAny<string>(), It.IsAny<FileStream>()).Result)
+                .Returns(new Uri(AUTHENTICATED_GIT_ARCHIVE_URL))
+                .Returns(new Uri(AUTHENTICATED_METADATA_ARCHIVE_URL));
+
+            _mockFileSystemProvider
+                .SetupSequence(m => m.GetTempFileName())
+                .Returns(GIT_ARCHIVE_FILE_PATH)
+                .Returns(METADATA_ARCHIVE_FILE_PATH);
+
+            _mockEnvironmentVariableProvider.Setup(m => m.SourceGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_SOURCE_PAT);
+            _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_TARGET_PAT);
+            _mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);
+
+            // Act
+            var args = new MigrateRepoCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                SourceRepo = SOURCE_REPO,
+                GithubTargetOrg = TARGET_ORG,
+                TargetRepo = TARGET_REPO,
+                TargetApiUrl = TARGET_API_URL,
+                GhesApiUrl = GHES_API_URL,
+                AzureStorageConnectionString = AZURE_CONNECTION_STRING,
+            };
+            await _handler.Handle(args);
+
+            // Assert
+            // Verify that GetArchiveMigrationUrl was called twice for git archive (once during generation, once during retry)
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID), Times.Exactly(2));
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID), Times.Once);
+
+            // Verify that DownloadToFile was called twice for git archive (original URL failed, fresh URL succeeded)
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(GIT_ARCHIVE_URL, GIT_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(freshGitArchiveUrl, GIT_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(METADATA_ARCHIVE_URL, METADATA_ARCHIVE_FILE_PATH), Times.Once);
+        }
+
+        [Fact]
+        public async Task Metadata_Archive_Download_Retries_On_404_Error()
+        {
+            // Arrange
+            var freshMetadataArchiveUrl = "https://example.com/2/fresh";
+
+            _mockTargetGithubApi.Setup(x => x.GetOrganizationId(TARGET_ORG).Result).Returns(GITHUB_ORG_ID);
+            _mockTargetGithubApi.Setup(x => x.CreateGhecMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID);
+            _mockTargetGithubApi.Setup(x => x.DoesOrgExist(TARGET_ORG).Result).Returns(true);
+            _mockTargetGithubApi.Setup(x => x.StartMigration(
+                MIGRATION_SOURCE_ID,
+                GHES_REPO_URL,
+                GITHUB_ORG_ID,
+                TARGET_REPO,
+                GITHUB_SOURCE_PAT,
+                GITHUB_TARGET_PAT,
+                AUTHENTICATED_GIT_ARCHIVE_URL,
+                AUTHENTICATED_METADATA_ARCHIVE_URL,
+                false,
+                null,
+                false).Result).Returns(MIGRATION_ID);
+            _mockTargetGithubApi.Setup(x => x.GetMigration(MIGRATION_ID).Result).Returns((State: RepositoryMigrationStatus.Succeeded, TARGET_REPO, 0, null, null));
+
+            _mockSourceGithubApi.Setup(x => x.StartGitArchiveGeneration(SOURCE_ORG, SOURCE_REPO).Result).Returns(GIT_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.StartMetadataArchiveGeneration(SOURCE_ORG, SOURCE_REPO, false, false).Result).Returns(METADATA_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, GIT_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, METADATA_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID).Result).Returns(GIT_ARCHIVE_URL);
+
+            // Setup GetArchiveMigrationUrl to return original URL first, then fresh URL on retry
+            _mockSourceGithubApi.SetupSequence(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID).Result)
+                .Returns(METADATA_ARCHIVE_URL)
+                .Returns(freshMetadataArchiveUrl);
+
+            _mockHttpDownloadService
+                .Setup(x => x.DownloadToFile(GIT_ARCHIVE_URL, GIT_ARCHIVE_FILE_PATH))
+                .Returns(Task.CompletedTask);
+
+            // Setup HttpDownloadService to fail first time with 404, succeed on retry
+            _mockHttpDownloadService
+                .SetupSequence(x => x.DownloadToFile(It.IsAny<string>(), METADATA_ARCHIVE_FILE_PATH))
+                .ThrowsAsync(CreateHttpNotFoundException())
+                .Returns(Task.CompletedTask);
+
+            _mockAzureApi
+                .SetupSequence(x => x.UploadToBlob(It.IsAny<string>(), It.IsAny<FileStream>()).Result)
+                .Returns(new Uri(AUTHENTICATED_GIT_ARCHIVE_URL))
+                .Returns(new Uri(AUTHENTICATED_METADATA_ARCHIVE_URL));
+
+            _mockFileSystemProvider
+                .SetupSequence(m => m.GetTempFileName())
+                .Returns(GIT_ARCHIVE_FILE_PATH)
+                .Returns(METADATA_ARCHIVE_FILE_PATH);
+
+            _mockEnvironmentVariableProvider.Setup(m => m.SourceGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_SOURCE_PAT);
+            _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_TARGET_PAT);
+            _mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);
+
+            // Act
+            var args = new MigrateRepoCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                SourceRepo = SOURCE_REPO,
+                GithubTargetOrg = TARGET_ORG,
+                TargetRepo = TARGET_REPO,
+                TargetApiUrl = TARGET_API_URL,
+                GhesApiUrl = GHES_API_URL,
+                AzureStorageConnectionString = AZURE_CONNECTION_STRING,
+            };
+            await _handler.Handle(args);
+
+            // Assert
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID), Times.Once);
+            // Verify that GetArchiveMigrationUrl was called twice for metadata archive (once during generation, once during retry)
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID), Times.Exactly(2));
+
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(GIT_ARCHIVE_URL, GIT_ARCHIVE_FILE_PATH), Times.Once);
+            // Verify that DownloadToFile was called twice for metadata archive (original URL failed, fresh URL succeeded)
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(METADATA_ARCHIVE_URL, METADATA_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(freshMetadataArchiveUrl, METADATA_ARCHIVE_FILE_PATH), Times.Once);
+        }
+
+        [Fact]
+        public async Task Both_Archives_Download_Retry_On_404_Error()
+        {
+            // Arrange
+            var freshGitArchiveUrl = "https://example.com/1/fresh";
+            var freshMetadataArchiveUrl = "https://example.com/2/fresh";
+
+            _mockTargetGithubApi.Setup(x => x.GetOrganizationId(TARGET_ORG).Result).Returns(GITHUB_ORG_ID);
+            _mockTargetGithubApi.Setup(x => x.CreateGhecMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID);
+            _mockTargetGithubApi.Setup(x => x.DoesOrgExist(TARGET_ORG).Result).Returns(true);
+            _mockTargetGithubApi.Setup(x => x.StartMigration(
+                MIGRATION_SOURCE_ID,
+                GHES_REPO_URL,
+                GITHUB_ORG_ID,
+                TARGET_REPO,
+                GITHUB_SOURCE_PAT,
+                GITHUB_TARGET_PAT,
+                AUTHENTICATED_GIT_ARCHIVE_URL,
+                AUTHENTICATED_METADATA_ARCHIVE_URL,
+                false,
+                null,
+                false).Result).Returns(MIGRATION_ID);
+            _mockTargetGithubApi.Setup(x => x.GetMigration(MIGRATION_ID).Result).Returns((State: RepositoryMigrationStatus.Succeeded, TARGET_REPO, 0, null, null));
+
+            _mockSourceGithubApi.Setup(x => x.StartGitArchiveGeneration(SOURCE_ORG, SOURCE_REPO).Result).Returns(GIT_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.StartMetadataArchiveGeneration(SOURCE_ORG, SOURCE_REPO, false, false).Result).Returns(METADATA_ARCHIVE_ID);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, GIT_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+            _mockSourceGithubApi.Setup(x => x.GetArchiveMigrationStatus(SOURCE_ORG, METADATA_ARCHIVE_ID).Result).Returns(ArchiveMigrationStatus.Exported);
+
+            // Setup GetArchiveMigrationUrl to return original URLs first, then fresh URLs on retry
+            _mockSourceGithubApi.SetupSequence(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID).Result)
+                .Returns(GIT_ARCHIVE_URL)
+                .Returns(freshGitArchiveUrl);
+
+            _mockSourceGithubApi.SetupSequence(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID).Result)
+                .Returns(METADATA_ARCHIVE_URL)
+                .Returns(freshMetadataArchiveUrl);
+
+            // Setup HttpDownloadService to fail first time for both archives with 404, succeed on retry
+            _mockHttpDownloadService
+                .SetupSequence(x => x.DownloadToFile(It.IsAny<string>(), GIT_ARCHIVE_FILE_PATH))
+                .ThrowsAsync(CreateHttpNotFoundException())
+                .Returns(Task.CompletedTask);
+
+            _mockHttpDownloadService
+                .SetupSequence(x => x.DownloadToFile(It.IsAny<string>(), METADATA_ARCHIVE_FILE_PATH))
+                .ThrowsAsync(CreateHttpNotFoundException())
+                .Returns(Task.CompletedTask);
+
+            _mockAzureApi
+                .SetupSequence(x => x.UploadToBlob(It.IsAny<string>(), It.IsAny<FileStream>()).Result)
+                .Returns(new Uri(AUTHENTICATED_GIT_ARCHIVE_URL))
+                .Returns(new Uri(AUTHENTICATED_METADATA_ARCHIVE_URL));
+
+            _mockFileSystemProvider
+                .SetupSequence(m => m.GetTempFileName())
+                .Returns(GIT_ARCHIVE_FILE_PATH)
+                .Returns(METADATA_ARCHIVE_FILE_PATH);
+
+            _mockEnvironmentVariableProvider.Setup(m => m.SourceGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_SOURCE_PAT);
+            _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny<bool>())).Returns(GITHUB_TARGET_PAT);
+            _mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);
+
+            // Act
+            var args = new MigrateRepoCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                SourceRepo = SOURCE_REPO,
+                GithubTargetOrg = TARGET_ORG,
+                TargetRepo = TARGET_REPO,
+                TargetApiUrl = TARGET_API_URL,
+                GhesApiUrl = GHES_API_URL,
+                AzureStorageConnectionString = AZURE_CONNECTION_STRING,
+            };
+            await _handler.Handle(args);
+
+            // Assert
+            // Verify that GetArchiveMigrationUrl was called twice for both archives (once during generation, once during retry)
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, GIT_ARCHIVE_ID), Times.Exactly(2));
+            _mockSourceGithubApi.Verify(x => x.GetArchiveMigrationUrl(SOURCE_ORG, METADATA_ARCHIVE_ID), Times.Exactly(2));
+
+            // Verify that DownloadToFile was called twice for each archive (original URL failed, fresh URL succeeded)
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(GIT_ARCHIVE_URL, GIT_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(freshGitArchiveUrl, GIT_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(METADATA_ARCHIVE_URL, METADATA_ARCHIVE_FILE_PATH), Times.Once);
+            _mockHttpDownloadService.Verify(x => x.DownloadToFile(freshMetadataArchiveUrl, METADATA_ARCHIVE_FILE_PATH), Times.Once);
+        }
+
         private static HttpRequestException CreateHttpForbiddenException()
         {
             // Use the constructor that sets the StatusCode property (available in .NET 5+)
             return new HttpRequestException("Response status code does not indicate success: 403 (Forbidden).", null, HttpStatusCode.Forbidden);
         }
+
+        private static HttpRequestException CreateHttpNotFoundException()
+        {
+            // Use the constructor that sets the StatusCode property (available in .NET 5+)
+            return new HttpRequestException("Response status code does not indicate success: 404 (Not Found).", null, HttpStatusCode.NotFound);
+        }
     }
 }
diff --git a/src/gei/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gei/Commands/MigrateRepo/MigrateRepoCommandHandler.cs
@@ -267,7 +267,7 @@ private string ExtractGhesBaseUrl(string ghesApiUrl)
             {
                 await _httpDownloadService.DownloadToFile(gitArchiveUrl, gitArchiveDownloadFilePath);
             }
-            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
+            catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.NotFound)
             {
                 // URL likely expired, regenerate and retry
                 _log.LogInformation("Git archive URL appears to have expired, regenerating fresh URL and retrying download...");
@@ -282,7 +282,7 @@ private string ExtractGhesBaseUrl(string ghesApiUrl)
             {
                 await _httpDownloadService.DownloadToFile(metadataArchiveUrl, metadataArchiveDownloadFilePath);
             }
-            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
+            catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.NotFound)
             {
                 // URL likely expired, regenerate and retry
                 _log.LogInformation("Metadata archive URL appears to have expired, regenerating fresh URL and retrying download...");

Original file line number	Diff line number	Diff line change
`@@ -267,7 +267,7 @@ private string ExtractGhesBaseUrl(string ghesApiUrl)`
`267`	`267`	`{`
`268`	`268`	`await _httpDownloadService.DownloadToFile(gitArchiveUrl, gitArchiveDownloadFilePath);`
`269`	`269`	`}`
`270`		`- catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)`
	`270`	`+ catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.NotFound)`
`271`	`271`	`{`
`272`	`272`	`// URL likely expired, regenerate and retry`
`273`	`273`	`_log.LogInformation("Git archive URL appears to have expired, regenerating fresh URL and retrying download...");`
`@@ -282,7 +282,7 @@ private string ExtractGhesBaseUrl(string ghesApiUrl)`
`282`	`282`	`{`
`283`	`283`	`await _httpDownloadService.DownloadToFile(metadataArchiveUrl, metadataArchiveDownloadFilePath);`
`284`	`284`	`}`
`285`		`- catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)`
	`285`	`+ catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.NotFound)`
`286`	`286`	`{`
`287`	`287`	`// URL likely expired, regenerate and retry`
`288`	`288`	`_log.LogInformation("Metadata archive URL appears to have expired, regenerating fresh URL and retrying download...");`