diff --git a/README.rst b/README.rst index c500cf7..c5fab28 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ were never merged. --gitlab-url http://workbench.dachary.org \ --gitlab-token sxQJ67SQKihMrGWVf \ --gitlab-repo ceph/ceph-backports \ - --github-token 64933d355fda9844aadd4e224d \ + --github-auth githubusername:64933d355fda9844aadd4e224d \ --github-repo ceph/ceph \ --ignore-closed @@ -90,7 +90,7 @@ Hacking --gitlab-url http://workbench.dachary.org \ --gitlab-token XXXXXXXXX \ --gitlab-repo dachary/testrepo2 \ - --github-token XXXXXXXXX \ + --github-auth XXXXXXXXX \ --github-repo dachary/testrepo \ --ssh-public-key ~/.ssh/id_rsa.pub \ --verbose diff --git a/github2gitlab/main.py b/github2gitlab/main.py index 1b2cc39..8552126 100644 --- a/github2gitlab/main.py +++ b/github2gitlab/main.py @@ -21,6 +21,7 @@ import git import gitdb import hashlib +from http.client import HTTPConnection import json import logging import os @@ -57,15 +58,18 @@ def __init__(self, args): if not self.args.gitlab_repo: self.args.gitlab_repo = self.args.github_repo - (self.args.gitlab_namespace, - self.args.gitlab_name) = self.args.gitlab_repo.split('/') + + repo_parts = self.args.gitlab_repo.split('/') + self.args.gitlab_name = repo_parts.pop() + self.args.gitlab_namespace = '/'.join(repo_parts) + self.args.gitlab_repo = parse.quote_plus(self.args.gitlab_repo) self.github = { 'url': "https://api.github.com", 'git': "https://github.com", 'repo': self.args.github_repo, - 'token': self.args.github_token, + 'auth': self.args.github_auth or None } if self.args.branches: self.github['branches'] = self.args.branches.split(',') @@ -74,6 +78,7 @@ def __init__(self, args): 'host': self.args.gitlab_url, 'name': self.args.gitlab_name, 'namespace': self.args.gitlab_namespace, + 'group': None, 'url': self.args.gitlab_url + "/api/v4", 'repo': self.args.gitlab_repo, 'token': self.args.gitlab_token, @@ -81,11 +86,21 @@ def __init__(self, args): if self.args.verbose: level = logging.DEBUG + HTTPConnection.debuglevel = 1 else: level = logging.INFO + logging.getLogger("urllib3").setLevel(level) logging.getLogger('github2gitlab').setLevel(level) + g = self.gitlab + url = g['url'] + "/groups" + query = {'private_token': g['token'], 'all_available': 'true', 'per_page': 10000} + groups = requests.get(url, params=query).json() + matches = list(filter(lambda group: group['full_path'] == g['namespace'].lower(), groups)) + if any(matches): + g['group_id'] = matches[0]['id'] + self.tmpdir = "/tmp" @staticmethod @@ -101,8 +116,8 @@ def get_parser(): required=True) parser.add_argument('--gitlab-repo', help='Gitlab repo (for instance ceph/ceph)') - parser.add_argument('--github-token', - help='GitHub authentication token') + parser.add_argument('--github-auth', + help='GitHub auth credentials, in the form username:token') parser.add_argument('--github-repo', help='GitHub repo (for instance ceph/ceph)', required=True) @@ -115,6 +130,9 @@ def get_parser(): parser.add_argument('--ignore-closed', action='store_const', const=True, help='ignore pull requests closed and not merged') + parser.add_argument('--skip-add-key', action='store_const', + const=True, + help='do not attempt to add local SSH key to Gitlab') parser.add_argument('--skip-pull-requests', action='store_const', const=True, help='do not mirror PR to MR') @@ -127,6 +145,9 @@ def get_parser(): parser.add_argument('--clean', action='store_const', const=True, help='Remove the repo after sync') + parser.add_argument('--visibility', + help='Visbility of created repos (public, internal, private)', + default='public') return parser @staticmethod @@ -134,7 +155,8 @@ def factory(argv): return GitHub2GitLab(GitHub2GitLab.get_parser().parse_args(argv)) def run(self): - self.add_key() + if not self.args.skip_add_key: + self.add_key() if self.add_project(): self.unprotect_branches() self.git_mirror() @@ -153,8 +175,7 @@ def sh(self, command): args=command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - shell=True, - bufsize=1) + shell=True) lines = [] with proc.stdout: for line in iter(proc.stdout.readline, b''): @@ -179,9 +200,15 @@ def gitlab_create_remote(self, repo): def git_mirror(self): name = self.gitlab['name'] + url = self.github['git'] + + if(self.github['auth']): + url = self.github['git'].replace('https://', 'https://{}@'.format(self.github['auth'])) + if not os.path.exists(name): - self.sh("git clone --bare " + self.github['git'] + + self.sh("git clone --bare " + url + "/" + self.github['repo'] + " " + name) + repo = git.Repo(name) os.chdir(name) if not hasattr(repo.remotes, 'gitlab'): @@ -219,7 +246,7 @@ def git_mirror(self): def git_mirror_optimize(self, repo): self.sh("git fetch origin +refs/pull/*:refs/remotes/origin/pull/*") for head in repo.refs: - pr = re.search('^origin/pull/(\d+)/head$', head.name) + pr = re.search('^origin/pull/(\\d+)/head$', head.name) if not pr: continue pr = pr.group(1) @@ -285,15 +312,21 @@ def add_project(self): g = self.gitlab url = g['url'] + "/projects/" + g['repo'] query = {'private_token': g['token']} + if g['group_id']: + query['group_id'] = g['group_id'] + if (requests.get(url, params=query).status_code == requests.codes.ok): log.debug("project " + url + " already exists") return None else: - log.info("add project " + g['repo']) url = g['url'] + "/projects" - query['public'] = 'true' - query['namespace'] = g['namespace'] + query['visibility'] = self.args.visibility query['name'] = g['name'] + if g['group_id']: + query['namespace_id'] = g['group_id'] + else: + query['namespace'] = g['namespace'] + log.info("add project " + g['repo']) result = requests.post(url, params=query) if result.status_code != requests.codes.created: raise ValueError(result.text) @@ -484,8 +517,8 @@ def get_pull_requests(self): "https://developer.github.com/v3/pulls/#list-pull-requests" g = self.github query = {'state': 'all'} - if self.args.github_token: - query['access_token'] = g['token'] + if g['auth']: + query['access_token'] = g['auth'].split(':')[1] def f(pull): if self.args.ignore_closed: @@ -511,7 +544,7 @@ def create_merge_request(self, query): g = self.gitlab query['private_token'] = g['token'] url = g['url'] + "/projects/" + g['repo'] + "/merge_requests" - log.info('create_merge_request: ' + str(query)) + log.debug('create_merge_request: ' + str(query)) result = requests.post(url, params=query) if result.status_code != requests.codes.created: raise ValueError(result.text) @@ -547,7 +580,7 @@ def put_merge_request(self, merge_request, updates): updates['private_token'] = g['token'] url = (g['url'] + "/projects/" + g['repo'] + "/merge_requests/" + str(merge_request['iid'])) - log.info('update_merge_request: ' + url + ' <= ' + str(updates)) + log.debug('update_merge_request: ' + url + ' <= ' + str(updates)) return requests.put(url, params=updates).json() def verify_merge_update(self, updates, result):