wip: support gh source url for ghdr source migrations

.github/workflows/CI.yml

@@ -169,7 +169,7 @@ jobs:
       fail-fast: false
       matrix:
         runner-os: [windows-latest, ubuntu-latest, macos-latest]
-        source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab]
+        source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab, GithubDR]
     runs-on: ${{ matrix.runner-os }}
     concurrency: integration-test-${{ matrix.source-vcs }}-${{ matrix.runner-os }}
     steps:
@@ -284,6 +284,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
           GITLAB_PAT: ${{ secrets.GITLAB_PAT }}
+          E2E_SOURCE_PROXIMA_PAT: ${{ secrets.E2E_SOURCE_PROXIMA_PAT }}
           LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native"
         run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9
 

.github/workflows/integration-tests.yml

@@ -94,7 +94,7 @@ jobs:
       fail-fast: false
       matrix:
         runner-os: [windows-latest, ubuntu-latest, macos-latest]
-        source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab]
+        source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab, GithubDR]
     runs-on: ${{ matrix.runner-os }}
     concurrency: integration-test-${{ matrix.source-vcs }}-${{ matrix.runner-os }}
     steps:
@@ -217,6 +217,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
         GITLAB_PAT: ${{ secrets.GITLAB_PAT }}
+        E2E_SOURCE_PROXIMA_PAT: ${{ secrets.E2E_SOURCE_PROXIMA_PAT }}
         GEI_DEBUG_MODE: 'true'
         LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native'
       run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9

README.md

@@ -53,6 +53,14 @@ To see the available commands and options run:
 
 Refer to the [official documentation](https://docs.github.com/en/migrations/using-github-enterprise-importer) for more details, including differences when migrating from GitHub Enterprise Server.
 
+#### Migrating from GitHub Enterprise Cloud with data residency (ghe.com)
+
+`gh gei` also supports migrating from a GitHub Enterprise Cloud with data residency tenant. Pass `--github-source-api-url` with the API endpoint of your source tenant (or set the `GH_SOURCE_API_URL` environment variable):
+
+>`gh gei migrate-repo --github-source-org SOURCE_ORG --source-repo SOURCE_REPO --github-source-api-url https://api.SUBDOMAIN.ghe.com --github-target-org TARGET_ORG --target-repo TARGET_REPO`
+
+If the target is also a data residency tenant, add `--target-api-url` and `--target-uploads-url` as usual. `--github-source-api-url` and `--ghes-api-url` cannot be used together.
+
 ### Azure DevOps to GitHub Usage
 1. Create Personal Access Tokens with access to the Azure DevOps org, and the GitHub org (for more details on scopes needed refer to our [official documentation](https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer)).
 

RELEASENOTES.md

@@ -1 +1 @@
-
+- Added support for migrating from GitHub Enterprise Cloud with data residency (ghe.com) as a source in `gh gei migrate-repo`, `generate-script`, `migrate-secret-alerts`, and `migrate-code-scanning-alerts`. Configure via `--github-source-api-url` (e.g. `https://api.tenant.ghe.com`) or the `GH_SOURCE_API_URL` environment variable. `--github-source-api-url` and `--ghes-api-url` cannot be used together.

src/Octoshift/Extensions/StringExtensions.cs

@@ -33,5 +33,8 @@ public static bool IsUrl(this string s)
                 && Uri.TryCreate(s, UriKind.Absolute, out var uri)
                 && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps);
         }
+
+        public static bool IsProximaApiUrl(this string s) =>
+            !s.IsNullOrWhiteSpace() && Regex.IsMatch(s.Trim(), @"^https?://api\.[^/]+\.ghe\.com/?$", RegexOptions.IgnoreCase);
     }
 }

src/Octoshift/Services/EnvironmentVariableProvider.cs

@@ -7,6 +7,7 @@ public class EnvironmentVariableProvider
 {
     private const string SOURCE_GH_PAT = "GH_SOURCE_PAT";
     private const string TARGET_GH_PAT = "GH_PAT";
+    private const string SOURCE_GH_API_URL = "GH_SOURCE_API_URL";
     private const string ADO_PAT = "ADO_PAT";
     private const string AZURE_STORAGE_CONNECTION_STRING = "AZURE_STORAGE_CONNECTION_STRING";
     private const string AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
@@ -31,6 +32,8 @@ public EnvironmentVariableProvider(OctoLogger logger)
     public virtual string SourceGithubPersonalAccessToken(bool throwIfNotFound = true) =>
         GetSecret(SOURCE_GH_PAT, false) ?? TargetGithubPersonalAccessToken(throwIfNotFound);
 
+    public virtual string SourceGithubApiUrl() => GetValue(SOURCE_GH_API_URL, false);
+
     public virtual string TargetGithubPersonalAccessToken(bool throwIfNotFound = true) =>
         GetSecret(TARGET_GH_PAT, throwIfNotFound);
 

src/OctoshiftCLI.IntegrationTests/GithubDRToGithub.cs

@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
+using OctoshiftCLI.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace OctoshiftCLI.IntegrationTests;
+
+// Integration test for GitHub Enterprise Cloud with data residency (GithubDR) source migrations.
+// Requires E2E_SOURCE_PROXIMA_PAT and GHEC_PAT secrets to be set. The source repo is treated as
+// read-only (tenant-managed); only the target org is reset each run.
+[Collection("Integration Tests")]
+public sealed class GithubDRToGithub : IDisposable
+{
+    private const string GITHUBDR_API_URL = "https://api.migration-tools-staffwus201.ghe.com";
+    private const string GITHUBDR_SOURCE_ORG = "octoshift";
+    private const string GITHUBDR_SOURCE_REPO = "tiny";
+    private const string UPLOADS_URL = "https://uploads.github.com";
+
+    private readonly ITestOutputHelper _output;
+    private readonly TestHelper _targetHelper;
+    private readonly HttpClient _versionClient;
+    private readonly HttpClient _targetGithubHttpClient;
+    private readonly GithubClient _targetGithubClient;
+    private readonly GithubApi _targetGithubApi;
+    private readonly HttpClient _sourceGithubHttpClient;
+    private readonly GithubClient _sourceGithubClient;
+    private readonly GithubApi _sourceGithubApi;
+    private readonly ArchiveUploader _archiveUploader;
+    private readonly Dictionary<string, string> _tokens;
+    private readonly DateTime _startTime;
+
+    public GithubDRToGithub(ITestOutputHelper output)
+    {
+        _startTime = DateTime.Now;
+        _output = output;
+
+        TestHelper.AssertCredentialsPresent(
+            ("E2E_SOURCE_PROXIMA_PAT", "GitHub Enterprise Cloud with data residency (source) personal access token"),
+            ("GHEC_PAT", "GitHub Enterprise Cloud (target) personal access token"));
+
+        var logger = new OctoLogger(_ => { }, x => _output.WriteLine(x), _ => { }, _ => { });
+
+        var sourceGithubToken = Environment.GetEnvironmentVariable("E2E_SOURCE_PROXIMA_PAT");
+        var targetGithubToken = Environment.GetEnvironmentVariable("GHEC_PAT");
+
+        _tokens = new Dictionary<string, string>
+        {
+            ["GH_SOURCE_PAT"] = sourceGithubToken,
+            ["GH_PAT"] = targetGithubToken,
+        };
+
+        _versionClient = new HttpClient();
+        var retryPolicy = new RetryPolicy(logger, "GithubDR (E2E_SOURCE_PROXIMA_PAT)");
+        var environmentVariableProvider = new EnvironmentVariableProvider(logger);
+
+        _sourceGithubHttpClient = new HttpClient();
+        _sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger, "GithubDR (E2E_SOURCE_PROXIMA_PAT)"), new DateTimeProvider(), sourceGithubToken);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy, environmentVariableProvider);
+        _sourceGithubApi = new GithubApi(_sourceGithubClient, GITHUBDR_API_URL, new RetryPolicy(logger, "GithubDR (E2E_SOURCE_PROXIMA_PAT)"), _archiveUploader);
+
+        _targetGithubHttpClient = new HttpClient();
+        _targetGithubClient = new GithubClient(logger, _targetGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger, "GitHub (GHEC_PAT)"), new DateTimeProvider(), targetGithubToken);
+        _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(logger, "GitHub (GHEC_PAT)"), _archiveUploader);
+
+        _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient);
+    }
+
+    [Fact]
+    public async Task Basic()
+    {
+        var githubTargetOrg = $"octoshift-e2e-githubdr-{TestHelper.GetOsName()}";
+
+        var retryPolicy = new RetryPolicy(null);
+
+        // Source repo is tenant-managed (read-only); only reset the target.
+        await retryPolicy.Retry(async () => await _targetHelper.ResetGithubTestEnvironment(githubTargetOrg));
+
+        var command = $"gei migrate-repo --github-source-org {GITHUBDR_SOURCE_ORG} --source-repo {GITHUBDR_SOURCE_REPO} --github-source-api-url {GITHUBDR_API_URL} --github-target-org {githubTargetOrg} --target-repo {GITHUBDR_SOURCE_REPO} --target-repo-visibility private --use-github-storage";
+
+        await _targetHelper.RunCliCommand(command, "gh", _tokens);
+
+        _targetHelper.AssertNoErrorInLogs(_startTime);
+
+        await _targetHelper.AssertGithubRepoExists(githubTargetOrg, GITHUBDR_SOURCE_REPO);
+        await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, GITHUBDR_SOURCE_REPO);
+    }
+
+    public void Dispose()
+    {
+        _sourceGithubHttpClient?.Dispose();
+        _targetGithubHttpClient?.Dispose();
+        _versionClient?.Dispose();
+    }
+}

src/OctoshiftCLI.Tests/StringExtensionsTests.cs

@@ -39,5 +39,27 @@ public void IsUrl_Detects_URLs_Correctly(string value, bool expectedResult)
 
             result.Should().Be(expectedResult);
         }
+
+        [Theory]
+        [InlineData("https://api.tenant.ghe.com", true)]
+        [InlineData("http://api.tenant.ghe.com", true)]
+        [InlineData("https://api.TENANT.ghe.com", true)]
+        [InlineData("https://api.tenant.ghe.com/", true)]
+        [InlineData("  https://api.tenant.ghe.com  ", true)]
+        [InlineData("https://api.foo.bar.ghe.com", true)]
+        [InlineData("https://tenant.ghe.com", false)]
+        [InlineData("https://api.github.com", false)]
+        [InlineData("https://api.tenant.ghe.com/foo", false)]
+        [InlineData("https://ghes.contoso.com/api/v3", false)]
+        [InlineData("api.tenant.ghe.com", false)]
+        [InlineData("", false)]
+        [InlineData(null, false)]
+        [InlineData("   ", false)]
+        public void IsProximaApiUrl_Detects_Proxima_Api_URLs_Correctly(string value, bool expectedResult)
+        {
+            var result = value.IsProximaApiUrl();
+
+            result.Should().Be(expectedResult);
+        }
     }
 }

src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs

@@ -244,6 +244,36 @@ public async Task Sequential_Github_Ghes_Repo()
             _script.Should().Be(expected);
         }
 
+        [Fact]
+        public async Task Sequential_Github_Proxima_Source_Emits_GithubSourceApiUrl_Flag()
+        {
+            // Arrange
+            const string proximaApiUrl = "https://api.tenant.ghe.com";
+
+            _mockGithubApi
+                .Setup(m => m.GetRepos(SOURCE_ORG))
+                .ReturnsAsync(new[] { (REPO, "private") });
+
+            var expected = $"Exec {{ gh gei migrate-repo --github-source-org \"{SOURCE_ORG}\" --source-repo \"{REPO}\" --github-target-org \"{TARGET_ORG}\" --target-repo \"{REPO}\" --github-source-api-url \"{proximaApiUrl}\" --target-repo-visibility private }}";
+
+            // Act
+            var args = new GenerateScriptCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                GithubTargetOrg = TARGET_ORG,
+                Output = new FileInfo("unit-test-output"),
+                GithubSourceApiUrl = proximaApiUrl,
+                NoSslVerify = true, // should be ignored for Proxima
+                Sequential = true
+            };
+            await _handler.Handle(args);
+
+            _script = TrimNonExecutableLines(_script);
+
+            // Assert
+            _script.Should().Be(expected);
+        }
+
         [Fact]
         public async Task Parallel_Github_Multiple_Repos()
         {

src/OctoshiftCLI.Tests/gei/Commands/GenerateScript/GenerateScriptCommandTests.cs

@@ -39,11 +39,12 @@ public void Should_Have_Options()
             var command = new GenerateScriptCommand();
             command.Should().NotBeNull();
             command.Name.Should().Be("generate-script");
-            command.Options.Count.Should().Be(17);
+            command.Options.Count.Should().Be(18);
 
             TestHelpers.VerifyCommandOption(command.Options, "github-source-org", true);
             TestHelpers.VerifyCommandOption(command.Options, "github-target-org", true);
             TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false);
+            TestHelpers.VerifyCommandOption(command.Options, "github-source-api-url", false);
             TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false);
             TestHelpers.VerifyCommandOption(command.Options, "skip-releases", false);
             TestHelpers.VerifyCommandOption(command.Options, "lock-source-repo", false);

src/OctoshiftCLI.Tests/gei/Commands/MigrateCodeScanningAlerts/MigrateCodeScanningAlertsCommandTests.cs

@@ -33,14 +33,15 @@ public void Should_Have_Options()
     {
         _command.Should().NotBeNull();
         _command.Name.Should().Be("migrate-code-scanning-alerts");
-        _command.Options.Count.Should().Be(11);
+        _command.Options.Count.Should().Be(12);
 
         TestHelpers.VerifyCommandOption(_command.Options, "source-org", true);
         TestHelpers.VerifyCommandOption(_command.Options, "source-repo", true);
         TestHelpers.VerifyCommandOption(_command.Options, "target-org", true);
         TestHelpers.VerifyCommandOption(_command.Options, "target-repo", false);
         TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false);
         TestHelpers.VerifyCommandOption(_command.Options, "ghes-api-url", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "github-source-api-url", false);
         TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false);
         TestHelpers.VerifyCommandOption(_command.Options, "github-source-pat", false);
         TestHelpers.VerifyCommandOption(_command.Options, "github-target-pat", false);

src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs

@@ -376,5 +376,63 @@ public void Validate_Throws_When_TargetRepo_Is_Url()
                 .ThrowExactly<OctoshiftCliException>()
                 .WithMessage("The --target-repo option expects a repository name, not a URL. Please provide just the repository name (e.g., 'my-repo' instead of 'https://github.com/my-org/my-repo').");
         }
+
+        [Fact]
+        public void GithubSourceApiUrl_And_GhesApiUrl_Both_Set_Throws()
+        {
+            var args = new MigrateRepoCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                SourceRepo = SOURCE_REPO,
+                GithubTargetOrg = TARGET_ORG,
+                GhesApiUrl = GHES_API_URL,
+                GithubSourceApiUrl = "https://api.tenant.ghe.com"
+            };
+
+            FluentActions.Invoking(() => args.Validate(_mockOctoLogger.Object))
+                .Should()
+                .ThrowExactly<OctoshiftCliException>()
+                .WithMessage("*Only one of --github-source-api-url or --ghes-api-url*");
+        }
+
+        [Fact]
+        public void UseGithubStorage_Validates_With_GithubSourceApiUrl()
+        {
+            var args = new MigrateRepoCommandArgs
+            {
+                GithubSourceOrg = SOURCE_ORG,
+                SourceRepo = SOURCE_REPO,
+                GithubTargetOrg = TARGET_ORG,
+                GithubSourceApiUrl = "https://api.tenant.ghe.com",
+                UseGithubStorage = true
+            };
+
+            args.Validate(_mockOctoLogger.Object);
+
+            args.TargetRepo.Should().Be(SOURCE_REPO);
+        }
+
+        [Fact]
+        public void GetSourceApiUrl_Prefers_GithubSourceApiUrl()
+        {
+            var args = new MigrateRepoCommandArgs
+            {
+                GhesApiUrl = GHES_API_URL,
+                GithubSourceApiUrl = "https://api.tenant.ghe.com"
+            };
+
+            args.GetSourceApiUrl().Should().Be("https://api.tenant.ghe.com");
+        }
+
+        [Fact]
+        public void GetSourceApiUrl_Falls_Back_To_GhesApiUrl()
+        {
+            var args = new MigrateRepoCommandArgs
+            {
+                GhesApiUrl = GHES_API_URL
+            };
+
+            args.GetSourceApiUrl().Should().Be(GHES_API_URL);
+        }
     }
 }

src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandTests.cs

@@ -13,7 +13,7 @@ public void Should_Have_Options()
 
             command.Should().NotBeNull();
             command.Name.Should().Be("migrate-repo");
-            command.Options.Count.Should().Be(27);
+            command.Options.Count.Should().Be(28);
 
             TestHelpers.VerifyCommandOption(command.Options, "github-source-org", true);
             TestHelpers.VerifyCommandOption(command.Options, "source-repo", true);
@@ -22,6 +22,7 @@ public void Should_Have_Options()
             TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false);
             TestHelpers.VerifyCommandOption(command.Options, "target-uploads-url", false);
             TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false);
+            TestHelpers.VerifyCommandOption(command.Options, "github-source-api-url", false);
             TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false);
             TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false);
             TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false);

src/OctoshiftCLI.Tests/gei/Commands/MigrateSecretAlerts/MigrateSecretAlertsCommandTests.cs

@@ -33,14 +33,15 @@ public void Should_Have_Options()
     {
         _command.Should().NotBeNull();
         _command.Name.Should().Be("migrate-secret-alerts");
-        _command.Options.Count.Should().Be(11);
+        _command.Options.Count.Should().Be(12);
 
         TestHelpers.VerifyCommandOption(_command.Options, "source-org", true);
         TestHelpers.VerifyCommandOption(_command.Options, "source-repo", true);
         TestHelpers.VerifyCommandOption(_command.Options, "target-org", true);
         TestHelpers.VerifyCommandOption(_command.Options, "target-repo", false);
         TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false);
         TestHelpers.VerifyCommandOption(_command.Options, "ghes-api-url", false);
+        TestHelpers.VerifyCommandOption(_command.Options, "github-source-api-url", false);
         TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false);
         TestHelpers.VerifyCommandOption(_command.Options, "github-source-pat", false);
         TestHelpers.VerifyCommandOption(_command.Options, "github-target-pat", false);

src/OctoshiftCLI.Tests/gei/Services/GhesVersionCheckerTests.cs

@@ -51,4 +51,12 @@ public async Task Empty_Ghes_Url_Returns_False()
         var result = await _service.AreBlobCredentialsRequired("");
         result.Should().Be(false);
     }
+
+    [Fact]
+    public async Task Proxima_Api_Url_Returns_False_Without_Calling_Source()
+    {
+        var result = await _service.AreBlobCredentialsRequired("https://api.tenant.ghe.com");
+        result.Should().Be(false);
+        _mockGithubApi.Verify(m => m.GetEnterpriseServerVersion(), Times.Never);
+    }
 }