diff --git a/CONFIG.rst b/CONFIG.rst index d3b702e..c5985d2 100644 --- a/CONFIG.rst +++ b/CONFIG.rst @@ -3,28 +3,6 @@ Config syntax This document describes the fields of the config file and their meaning. -patchwork -========= - -Section configuring the patchwork connection. - -archive -------- - -Patchwork's own ``/mbox/`` endpoints stopped working after one of its upgrades, -so patchwork is only used for metadata and the messages themselves are fetched -from the mailing list archive. - -``archive`` is the base URL of that archive, the message id gets appended to it -(default: ``https://lore.kernel.org/all``). - -user-agent ----------- - -The ``User-Agent`` header for all HTTP requests. lore.kernel.org rejects -requests from unknown agents with a 403, so this must be set to one of the -strings the archive recognizes. - poller ====== diff --git a/mailbot.py b/mailbot.py index ea6d400..7192b9c 100755 --- a/mailbot.py +++ b/mailbot.py @@ -382,7 +382,9 @@ def _resolve_thread(self, pw): self._series_id = pw_obj[0]['series'][0]['id'] - data = pw.get_mbox_by_msgid(mid) + r = requests.get(f'https://lore.kernel.org/all/{mid}/raw', + headers=http_headers) + data = r.content.decode('utf-8') msg = email.message_from_string(data, policy=default) self._series_author = msg.get('From') diff --git a/pw/patchwork.py b/pw/patchwork.py index 87beb51..9705f4f 100644 --- a/pw/patchwork.py +++ b/pw/patchwork.py @@ -29,60 +29,17 @@ class PatchworkPostException(Exception): pass -class PatchworkFetchException(Exception): - pass - - -def series_patches_ordered(series): - """Return the patches of a series in the order they should be applied - - Patchwork lists them in arrival order, use the n/total counter it parsed - out of the subject to put them back into the order the author intended. - """ - patches = series['patches'] - total = series['total'] - if total != len(patches): - core.log("Patch order - count does not add up?!", "") - return patches - - ordered = list(patches) - for i in range(total): - found = False - name = patches[i]['name'] - for j in range(total): - # scanning PW-parsed name - tags are separated by commas - if name.find(f" {j + 1}/{total}") >= 0 or \ - name.find(f",{j + 1}/{total}") >= 0 or \ - name.find(f"[{j + 1}/{total}") >= 0 or \ - name.find(f"0{j + 1}/{total}") >= 0: - if ordered[j] is not patches[i]: - core.log(f"Patch order - reordering {i} => {j + 1}") - ordered[j] = patches[i] - found = True - break - if not found: - core.log("Patch order - not all patches were found!", "") - return patches - return ordered - - class Patchwork(object): - # Patchwork mbox object types vs the names of the REST collections - _mbox_apis = {'cover': 'covers', 'patch': 'patches'} - def __init__(self, config): self._session = requests.Session() allowed_methods = Retry.DEFAULT_ALLOWED_METHODS | {'POST', 'PATCH'} - retry = Retry(connect=10, status=10, - status_forcelist={404, 429, 502, 503, 504}, + retry = Retry(connect=10, status=10, status_forcelist={502, 504}, allowed_methods=allowed_methods, backoff_factor=1) adapter = HTTPAdapter(max_retries=retry) self._session.mount('http://', adapter) self._session.mount('https://', adapter) self.server = config.get('patchwork', 'server') - self.archive = config.get('patchwork', 'archive', - fallback='https://lore.kernel.org/all').rstrip('/') ssl = config.getboolean('patchwork', 'use_ssl', fallback=True) self._proto = "https://" if ssl else "http://" self._token = config.get('patchwork', 'token', fallback='') @@ -113,7 +70,7 @@ def _request(self, url): try: core.log("Response data", ret.json()) except json.decoder.JSONDecodeError: - core.log("Response data", ret.content.decode('utf-8', 'replace')) + core.log("Response data", ret.content.decode()) finally: end = datetime.datetime.now() core.log("Response time GET (sec)", (end - start).total_seconds()) @@ -179,30 +136,12 @@ def get_by_msgid(self, object_type, msgid): msgid = urllib.parse.quote(msgid) return self._get(f'{object_type}/?msgid={msgid}&project={self._project}', api='').json() - # Patchwork's own /mbox/ endpoints have been serving empty responses ever - # since one of its upgrades, so the messages come from the list archive. - # Patchwork is only asked for the message ids. Note that the archive - # requires a well-known user-agent, see the 'user-agent' config option. - def get_mbox_by_msgid(self, msgid): - url = f'{self.archive}/{urllib.parse.quote(msgid.strip("<>"))}/raw' - ret = self._request(url) - if ret.status_code != 200: - raise PatchworkFetchException(url, ret) - # Archives serve the message as it was posted, which is not necessarily - # valid UTF-8. Losing a character beats blowing up the entire series. - return ret.content.decode('utf-8', 'replace') - - # Like patchwork's series mbox this contains the patches only, the cover - # letter is not part of it. - def series_to_mbox(self, series): - return ''.join([self.get_mbox_by_msgid(p['msgid']) - for p in series_patches_ordered(series)]) + def get_mbox_direct(self, url): + return self._request(url).content.decode() def get_mbox(self, object_type, identifier): - if object_type == 'series': - return self.series_to_mbox(self.get('series', identifier)) - obj = self.get(self._mbox_apis[object_type], identifier) - return self.get_mbox_by_msgid(obj['msgid']) + url = f'{self._proto}{self.server}/{object_type}/{identifier}/mbox/' + return self._request(url).content.decode() def _get(self, req, api='1.1'): if api: diff --git a/pw/pw_series.py b/pw/pw_series.py index 7084037..fdbee78 100644 --- a/pw/pw_series.py +++ b/pw/pw_series.py @@ -7,7 +7,6 @@ from core import Series from core import Patch from core import log, log_open_sec, log_end_sec -from .patchwork import series_patches_ordered # TODO: document @@ -22,7 +21,7 @@ def __init__(self, pw, pw_series): self.pull_url = None if pw_series['cover_letter']: - pw_cover_letter = pw.get_mbox_by_msgid(pw_series['cover_letter']['msgid']) + pw_cover_letter = pw.get_mbox('cover', pw_series['cover_letter']['id']) self.set_cover_letter(pw_cover_letter) elif self.pw_series['patches']: self.subject = self.pw_series['patches'][0]['name'] @@ -36,14 +35,44 @@ def __init__(self, pw, pw_series): # Fast path incomplete series if not pw_series['received_all']: for p in self.pw_series['patches']: - raw_patch = pw.get_mbox_by_msgid(p['msgid']) + raw_patch = pw.get_mbox('patch', p['id']) self.patches.append(Patch(raw_patch, p['id'])) return # Do more magic around series which are complete - for p in series_patches_ordered(self.pw_series): - raw_patch = pw.get_mbox_by_msgid(p['msgid']) - self.add_patch(Patch(raw_patch, p['id'])) + # Patchwork 2.2.2 orders them by arrival time + pids = [] + for p in self.pw_series['patches']: + pids.append(p['id']) + total = self.pw_series['total'] + if total == len(self.pw_series['patches']): + for i in range(total): + found = False + name = self.pw_series['patches'][i]['name'] + pid = self.pw_series['patches'][i]['id'] + for j in range(total): + # scanning PW-parsed name - tags are separated by commas + if name.find(f" {j + 1}/{total}") >= 0 or \ + name.find(f",{j + 1}/{total}") >= 0 or \ + name.find(f"[{j + 1}/{total}") >= 0 or \ + name.find(f"0{j + 1}/{total}") >= 0: + if pids[j] != pid: + log(f"Patch order - reordering {i} => {j + 1}") + pids[j] = pid + found = True + break + if not found: + log("Patch order - not all patches were found!", "") + pids = [] + for p in self.pw_series['patches']: + pids.append(p['id']) + break + else: + log("Patch order - count does not add up?!", "") + + for pid in pids: + raw_patch = pw.get_mbox('patch', pid) + self.add_patch(Patch(raw_patch, pid)) if not pw_series['cover_letter']: if len(self.patches) == 1: diff --git a/pw_brancher.py b/pw_brancher.py index b2013e6..7e5c79f 100755 --- a/pw_brancher.py +++ b/pw_brancher.py @@ -178,7 +178,8 @@ def apply_pending_patches(pw, config, tree, branch_name) -> Tuple[List, List]: else: log_open_sec("Applying: " + entry["series"][0]["name"]) seen_series.add(series_id) - data = pw.get_mbox('series', series_id) + mbox_url = entry["series"][0]["mbox"] + data = pw.get_mbox_direct(mbox_url) p = Patch(data) try: tree.apply(p)