diff --git a/Dockerfile b/Dockerfile index 2c28829..3f38010 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN apk add --no-cache \ ca-certificates \ git \ git-lfs \ + openssl \ && addgroup -g 1000 appuser \ && adduser -D -u 1000 -G appuser appuser diff --git a/README.rst b/README.rst index 10b9bf5..56a2e96 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,10 @@ Show the CLI help output:: CLI Help output:: github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [--token-from-gh] - [-q] [--as-app] [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] + [-q] [--as-app] [--app-id APP_ID] + [--app-private-key APP_PRIVATE_KEY] + [--app-installation-id APP_INSTALLATION_ID] + [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--incremental-by-files] [--starred] [--all-starred] [--starred-skip-size-over MB] [--watched] [--followers] [--following] [--all] @@ -78,6 +81,14 @@ CLI Help output:: -q, --quiet supress log messages less severe than warning, e.g. info --as-app authenticate as github app instead of as a user. + --app-id APP_ID github app id or client id; creates and refreshes + installation tokens (implies --as-app) + --app-private-key APP_PRIVATE_KEY + path to the github app private key (.pem), optionally + as file://... + --app-installation-id APP_INSTALLATION_ID + id of the github app installation to back up + (default: looked up from the user or organization) -o, --output-directory OUTPUT_DIRECTORY directory at which to backup the repositories -l, --log-level LOG_LEVEL @@ -212,9 +223,19 @@ For a user account (no ``--organization``), ``--as-app`` lists the repositories 7. Go to *Install App* in your app's settings 8. Select the account/organization and which repositories to back up -**CI/CD usage with GitHub Actions:** +**Using the app's private key:** -Store the App ID as a repository variable and the private key contents as a secret, then use ``actions/create-github-app-token``:: +Pass the App ID and private key file, and github-backup creates installation tokens itself, refreshing them before they expire so long-running backups keep working:: + + github-backup myorg --organization --app-id 12345 --app-private-key /path/to/app.private-key.pem -o ./backup --all + +The app's installation on the user or organization being backed up is looked up automatically. To skip the lookup, pass ``--app-installation-id`` with the number at the end of the installation's settings URL (``.../settings/installations/``). + +``--app-id`` implies ``--as-app`` and cannot be combined with ``-t``, ``-f``, ``--token-from-gh`` or the keychain options. The ``iss`` claim of the signed JWT is set to the value of ``--app-id``, so the app's client ID works there too. The JWT is signed by the ``openssl`` command line tool, which must be on your ``PATH`` (the Docker image includes it). + +**Using an existing installation token:** + +If something else already creates the installation token, pass it with ``-t`` and ``--as-app``. In GitHub Actions, store the App ID as a repository variable and the private key contents as a secret, then use ``actions/create-github-app-token``:: - uses: actions/create-github-app-token@v1 id: app-token @@ -224,7 +245,7 @@ Store the App ID as a repository variable and the private key contents as a secr - run: github-backup myorg -t ${{ steps.app-token.outputs.token }} --as-app -o ./backup --all -Note: Installation tokens expire after 1 hour. For long-running backups, use a fine-grained personal access token instead. +Note: Installation tokens passed with ``-t`` expire after 1 hour and are not refreshed. For backups that may run longer, use ``--app-private-key`` instead. Prefer SSH diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index d72317f..64c08c3 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -12,6 +12,7 @@ import platform import random import re +import shutil import socket import ssl import subprocess @@ -45,6 +46,7 @@ ) FILE_URI_PREFIX = "file://" +APP_TOKEN_REFRESH_MARGIN = 10 * 60 # seconds logger = logging.getLogger(__name__) @@ -195,6 +197,21 @@ def parse_args(args=None): dest="as_app", help="authenticate as github app instead of as a user.", ) + parser.add_argument( + "--app-id", + dest="app_id", + help="github app id or client id; creates and refreshes installation tokens (implies --as-app)", + ) + parser.add_argument( + "--app-private-key", + dest="app_private_key", + help="path to the github app private key (.pem), optionally as file://...", + ) + parser.add_argument( + "--app-installation-id", + dest="app_installation_id", + help="id of the github app installation to back up (default: looked up from the user or organization)", + ) parser.add_argument( "-o", "--output-directory", @@ -522,12 +539,39 @@ def parse_args(args=None): default=5, help="maximum number of retries for API calls (default: 5)", ) - return parser.parse_args(args) + args = parser.parse_args(args) + + if args.app_id or args.app_private_key or args.app_installation_id: + if not (args.app_id and args.app_private_key): + parser.error("--app-id and --app-private-key must be used together") + if ( + args.token_classic + or args.token_fine + or args.token_from_gh + or args.osx_keychain_item_name + or args.osx_keychain_item_account + ): + parser.error( + "--app-id cannot be combined with -t, -f, --token-from-gh or keychain options" + ) + args.app_private_key = args.app_private_key.removeprefix(FILE_URI_PREFIX) + if not os.path.isfile(args.app_private_key): + parser.error( + "--app-private-key file not found: {0}".format(args.app_private_key) + ) + if not shutil.which("openssl"): + parser.error("--app-private-key requires the 'openssl' command on PATH") + args.as_app = True + + return args def get_auth(args, encode=True, for_git_cli=False): auth = None + if args.app_id: + args.token_classic = get_app_installation_token(args) + if args.osx_keychain_item_name: if not args.osx_keychain_item_account: raise Exception( @@ -676,6 +720,110 @@ def read_token_from_gh_cli(args): return token +def build_app_jwt(private_key_path, claims): + """Build an RS256 JWT for authenticating as a GitHub App. + + Signs with the openssl CLI so the project keeps no runtime dependencies. + """ + + def b64url(data): + return base64.urlsafe_b64encode(data).rstrip(b"=") + + header = b64url(json.dumps({"alg": "RS256", "typ": "JWT"}).encode("ascii")) + signing_input = header + b"." + b64url(json.dumps(claims).encode("ascii")) + + try: + signature = subprocess.run( + ["openssl", "dgst", "-sha256", "-sign", private_key_path], + input=signing_input, + capture_output=True, + check=True, + ).stdout + except subprocess.CalledProcessError as e: + stderr = e.stderr.decode("utf-8", errors="replace").strip() + raise Exception("Unable to sign GitHub App JWT: {0}".format(stderr)) + + return (signing_input + b"." + b64url(signature)).decode("ascii") + + +def get_app_installation_token(args): + """Return a GitHub App installation token, creating a new one when needed. + + Installation tokens expire after an hour, so a new one is created once the + cached token is within APP_TOKEN_REFRESH_MARGIN seconds of expiring. + """ + cached_token = getattr(args, "_app_token_value", None) + if cached_token and time.time() < ( + args._app_token_expires_at - APP_TOKEN_REFRESH_MARGIN + ): + return cached_token + + # GitHub rejects an exp more than 10 minutes out; backdate iat to tolerate + # clock drift. + now = int(time.time()) + app_jwt = build_app_jwt( + args.app_private_key, {"iat": now - 60, "exp": now + 540, "iss": args.app_id} + ) + + if not args.app_installation_id: + owner_type = "orgs" if args.organization else "users" + installation = request_as_app( + args, + app_jwt, + "/{0}/{1}/installation".format(owner_type, args.user), + "Unable to find the GitHub App installation for {0}".format(args.user), + ) + args.app_installation_id = str(installation["id"]) + logger.info( + "Using GitHub App installation {0} for {1}".format( + args.app_installation_id, args.user + ) + ) + + response = request_as_app( + args, + app_jwt, + "/app/installations/{0}/access_tokens".format(args.app_installation_id), + "Unable to create GitHub App installation token", + method="POST", + ) + + args._app_token_value = response["token"] + args._app_token_expires_at = calendar.timegm( + time.strptime(response["expires_at"], "%Y-%m-%dT%H:%M:%SZ") + ) + logger.info( + "Created GitHub App installation token (expires {0})".format( + response["expires_at"] + ) + ) + return args._app_token_value + + +def request_as_app(args, app_jwt, path, error_message, method="GET"): + """Call the GitHub API authenticated as the app itself, with its JWT.""" + request = Request( + "https://{0}{1}".format(get_github_api_host(args), path), + data=b"" if method == "POST" else None, + method=method, + ) + request.add_header("Accept", "application/vnd.github+json") + request.add_header("Authorization", "Bearer " + app_jwt) + + try: + http_response = make_request_with_retry(request, app_jwt, args.max_retries) + except HTTPError as exc: + raise Exception( + "{0}: HTTP {1} {2}: {3}".format( + error_message, + exc.code, + exc.reason, + exc.read().decode("utf-8", errors="replace"), + ) + ) + return json.loads(http_response.read().decode("utf-8")) + + def get_github_repo_url(args, repository): if repository.get("is_gist"): if args.prefer_ssh: @@ -734,7 +882,6 @@ def retrieve_data(args, template, query_args=None, paginated=True, lazy=False): Handles throttling, retries, read errors, and DMCA takedowns. """ query_args = query_args or {} - auth = get_auth(args, encode=not args.as_app) per_page = 100 def _extract_next_page_url(link_header): @@ -758,6 +905,8 @@ def _extract_legal_url(response_body_bytes): # FIRST: Fetch response for attempt in range(args.max_retries + 1): + # Resolved per request so long runs pick up refreshed app tokens + auth = get_auth(args, encode=not args.as_app) request = _construct_request( per_page=per_page if paginated else None, query_args=query_args, @@ -2210,7 +2359,8 @@ def backup_repositories(args, output_directory, repositories): if repository["has_wiki"] and download_wiki: fetch_repository( repository["name"], - repo_url.replace(".git", ".wiki.git"), + # Rebuilt so a long repository clone can't leave it an expired app token + get_github_repo_url(args, repository).replace(".git", ".wiki.git"), os.path.join(repo_cwd, "wiki"), skip_existing=args.skip_existing, bare_clone=args.bare_clone, diff --git a/tests/test_app_auth.py b/tests/test_app_auth.py new file mode 100644 index 0000000..a206bc6 --- /dev/null +++ b/tests/test_app_auth.py @@ -0,0 +1,339 @@ +"""Tests for GitHub App authentication with an app private key.""" + +import base64 +import json +import shutil +import subprocess +import time +from io import BytesIO +from unittest.mock import patch +from urllib.error import HTTPError + +import pytest + +from github_backup import github_backup + + +class MockHTTPResponse: + def __init__(self, data, link_header=None): + self._content = json.dumps(data).encode("utf-8") + self.headers = {"x-ratelimit-remaining": "5000"} + if link_header: + self.headers["Link"] = link_header + + def getcode(self): + return 200 + + def read(self): + return self._content + + +def b64url_decode(segment): + return base64.urlsafe_b64decode(segment + "=" * (-len(segment) % 4)) + + +def expires_in(seconds): + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + seconds)) + + +@pytest.fixture +def key_file(tmp_path): + path = tmp_path / "app.pem" + path.write_text("key") + return str(path) + + +@pytest.fixture +def app_flags(key_file): + return ["--app-id", "12345", "--app-private-key", key_file] + + +@pytest.fixture +def app_args(app_flags): + return github_backup.parse_args( + app_flags + ["--app-installation-id", "678", "testorg"] + ) + + +class TokenEndpoint: + """Stands in for GET /{users,orgs}/{owner}/installation and + POST /app/installations/{id}/access_tokens.""" + + def __init__(self): + self.lookups = [] + self.requests = [] + self.expires_at = expires_in(3600) + + def __call__(self, request, *args, **kwargs): + if request.full_url.endswith("/installation"): + self.lookups.append(request) + return MockHTTPResponse({"id": 999}) + self.requests.append(request) + token = "ghs_{0}".format(len(self.requests)) + return MockHTTPResponse({"token": token, "expires_at": self.expires_at}) + + +@pytest.fixture +def token_endpoint(): + endpoint = TokenEndpoint() + with patch("github_backup.github_backup.build_app_jwt", return_value="app.jwt"): + with patch("github_backup.github_backup.urlopen", side_effect=endpoint): + yield endpoint + + +def test_app_flags_imply_as_app(app_args): + assert app_args.as_app is True + assert app_args.app_id == "12345" + assert app_args.app_installation_id == "678" + + +@pytest.mark.parametrize( + "flags", + [["--app-id", "12345"], ["--app-installation-id", "678"]], +) +def test_app_flags_must_be_used_together(flags, capsys): + with pytest.raises(SystemExit): + github_backup.parse_args(flags + ["testorg"]) + + assert "must be used together" in capsys.readouterr().err + + +def test_app_installation_id_is_optional(app_flags): + args = github_backup.parse_args(app_flags + ["testorg"]) + + assert args.as_app is True + assert args.app_installation_id is None + + +def test_file_uri_private_key_path(key_file): + args = github_backup.parse_args( + ["--app-id", "12345", "--app-private-key", "file://" + key_file, "testorg"] + ) + + assert args.app_private_key == key_file + + +def test_missing_private_key_fails_early(tmp_path, capsys): + missing = str(tmp_path / "missing.pem") + with pytest.raises(SystemExit): + github_backup.parse_args( + ["--app-id", "12345", "--app-private-key", missing, "testorg"] + ) + + assert "file not found: " + missing in capsys.readouterr().err + + +def test_missing_openssl_fails_early(app_flags, capsys): + with patch("github_backup.github_backup.shutil.which", return_value=None): + with pytest.raises(SystemExit): + github_backup.parse_args(app_flags + ["testorg"]) + + assert "requires the 'openssl' command" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "token_args", + [ + ["-t", "ghp_token"], + ["-f", "github_pat_token"], + ["--token-from-gh"], + ["--keychain-name", "name", "--keychain-account", "account"], + ], +) +def test_app_flags_reject_other_token_sources(app_flags, token_args, capsys): + with pytest.raises(SystemExit): + github_backup.parse_args(app_flags + token_args + ["testorg"]) + + assert "cannot be combined" in capsys.readouterr().err + + +@pytest.mark.skipif(shutil.which("openssl") is None, reason="openssl not installed") +def test_build_app_jwt_is_verifiable_rs256(tmp_path): + private_key = tmp_path / "app.pem" + public_key = tmp_path / "app.pub" + subprocess.run( + ["openssl", "genpkey", "-algorithm", "RSA", "-out", str(private_key)], + check=True, + capture_output=True, + ) + subprocess.run( + [ + "openssl", + "pkey", + "-in", + str(private_key), + "-pubout", + "-out", + str(public_key), + ], + check=True, + capture_output=True, + ) + + app_jwt = github_backup.build_app_jwt(str(private_key), {"iss": "12345"}) + + header, payload, signature = app_jwt.split(".") + assert json.loads(b64url_decode(header)) == {"alg": "RS256", "typ": "JWT"} + assert json.loads(b64url_decode(payload)) == {"iss": "12345"} + (tmp_path / "sig").write_bytes(b64url_decode(signature)) + verify = subprocess.run( + ["openssl", "dgst", "-sha256", "-verify", str(public_key)] + + ["-signature", str(tmp_path / "sig")], + input="{0}.{1}".format(header, payload).encode("ascii"), + capture_output=True, + ) + assert verify.returncode == 0, verify.stdout + verify.stderr + + +def test_build_app_jwt_reports_openssl_errors(): + error = subprocess.CalledProcessError(1, "openssl", stderr=b"Could not read key") + with patch("github_backup.github_backup.subprocess.run", side_effect=error): + with pytest.raises(Exception, match="Could not read key"): + github_backup.build_app_jwt("app.pem", {}) + + +def test_jwt_claims(app_args): + with patch( + "github_backup.github_backup.build_app_jwt", return_value="app.jwt" + ) as mock_build, patch( + "github_backup.github_backup.urlopen", + return_value=MockHTTPResponse( + {"token": "ghs_1", "expires_at": expires_in(3600)} + ), + ): + now = int(time.time()) + github_backup.get_auth(app_args) + + key_path, claims = mock_build.call_args.args + assert key_path == app_args.app_private_key + assert claims["iss"] == "12345" + assert now - 61 <= claims["iat"] <= now - 59 + assert now + 539 <= claims["exp"] <= now + 541 + + +def test_get_auth_creates_installation_token(app_args, token_endpoint): + assert github_backup.get_auth(app_args, encode=False) == "ghs_1" + + request = token_endpoint.requests[0] + assert request.get_method() == "POST" + assert ( + request.full_url == "https://api.github.com/app/installations/678/access_tokens" + ) + assert request.get_header("Authorization") == "Bearer app.jwt" + + +def test_installation_token_on_enterprise_host(app_args, token_endpoint): + app_args.github_host = "ghe.example.com" + + github_backup.get_auth(app_args) + + assert token_endpoint.requests[0].full_url == ( + "https://ghe.example.com/api/v3/app/installations/678/access_tokens" + ) + + +def test_installation_token_is_cached(app_args, token_endpoint): + assert github_backup.get_auth(app_args, encode=False) == "ghs_1" + assert github_backup.get_auth(app_args, encode=False) == "ghs_1" + + assert len(token_endpoint.requests) == 1 + + +def test_installation_token_is_refreshed_before_expiry(app_args, token_endpoint): + token_endpoint.expires_at = expires_in(github_backup.APP_TOKEN_REFRESH_MARGIN - 60) + + assert github_backup.get_auth(app_args, encode=False) == "ghs_1" + assert github_backup.get_auth(app_args, encode=False) == "ghs_2" + + +@pytest.mark.parametrize( + "owner_flags, lookup_url", + [ + ([], "https://api.github.com/users/testorg/installation"), + (["--organization"], "https://api.github.com/orgs/testorg/installation"), + ], +) +def test_installation_is_looked_up_once( + app_flags, owner_flags, lookup_url, token_endpoint +): + args = github_backup.parse_args(app_flags + owner_flags + ["testorg"]) + token_endpoint.expires_at = expires_in(github_backup.APP_TOKEN_REFRESH_MARGIN - 60) + + github_backup.get_auth(args) + github_backup.get_auth(args) + + assert [r.full_url for r in token_endpoint.lookups] == [lookup_url] + assert token_endpoint.lookups[0].get_header("Authorization") == "Bearer app.jwt" + assert [r.full_url for r in token_endpoint.requests] == [ + "https://api.github.com/app/installations/999/access_tokens" + ] * 2 + + +def test_installation_lookup_http_error(app_flags): + args = github_backup.parse_args(app_flags + ["testorg"]) + error = HTTPError( + "https://api.github.com", + 404, + "Not Found", + {}, + BytesIO(b'{"message":"Not Found"}'), + ) + with patch("github_backup.github_backup.build_app_jwt", return_value="app.jwt"): + with patch("github_backup.github_backup.urlopen", side_effect=error): + with pytest.raises( + Exception, + match="Unable to find the GitHub App installation for testorg: HTTP 404", + ): + github_backup.get_auth(args) + + +def test_git_url_uses_installation_token(app_args, token_endpoint): + repository = {"owner": {"login": "testorg"}, "name": "repo"} + + assert github_backup.get_github_repo_url(app_args, repository) == ( + "https://x-access-token:ghs_1@github.com/testorg/repo.git" + ) + + +def test_installation_token_http_error(app_args): + error = HTTPError( + "https://api.github.com", + 401, + "Unauthorized", + {}, + BytesIO(b'{"message":"Integration not found"}'), + ) + with patch("github_backup.github_backup.build_app_jwt", return_value="app.jwt"): + with patch("github_backup.github_backup.urlopen", side_effect=error): + with pytest.raises( + Exception, match="HTTP 401 Unauthorized: .*Integration not found" + ): + github_backup.get_auth(app_args) + + +def test_pagination_picks_up_refreshed_token(app_args): + """A long paginated listing keeps working across a token refresh.""" + pages = [ + MockHTTPResponse( + [{"id": 1}], + link_header='; rel="next"', + ), + MockHTTPResponse([{"id": 2}]), + ] + tokens = iter(["ghs_1", "ghs_2"]) + page_auth = [] + + def mock_urlopen(request, *args, **kwargs): + page_auth.append(request.get_header("Authorization")) + return pages[len(page_auth) - 1] + + with patch( + "github_backup.github_backup.get_app_installation_token", + side_effect=lambda args: next(tokens), + ), patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen): + results = github_backup.retrieve_data( + app_args, "https://api.github.com/orgs/testorg/repos" + ) + + assert results == [{"id": 1}, {"id": 2}] + assert page_auth == [b"token ghs_1", b"token ghs_2"]