Skip to content

Revert "pw: fetch the patches from lore instead of patchwork" #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matttbe wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Revert "pw: fetch the patches from lore instead of patchwork" #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 0 additions & 22 deletions CONFIG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,6 @@ Config syntax

This document describes the fields of the config file and their meaning.

patchwork
=========

Section configuring the patchwork connection.

archive
-------

Patchwork's own ``/mbox/`` endpoints stopped working after one of its upgrades,
so patchwork is only used for metadata and the messages themselves are fetched
from the mailing list archive.

``archive`` is the base URL of that archive, the message id gets appended to it
(default: ``https://lore.kernel.org/all``).

user-agent
----------

The ``User-Agent`` header for all HTTP requests. lore.kernel.org rejects
requests from unknown agents with a 403, so this must be set to one of the
strings the archive recognizes.

poller
======

Expand Down
4 changes: 3 additions & 1 deletion mailbot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,9 @@ def _resolve_thread(self, pw):

self._series_id = pw_obj[0]['series'][0]['id']

data = pw.get_mbox_by_msgid(mid)
r = requests.get(f'https://lore.kernel.org/all/{mid}/raw',
headers=http_headers)
data = r.content.decode('utf-8')
msg = email.message_from_string(data, policy=default)
self._series_author = msg.get('From')

Expand Down
73 changes: 6 additions & 67 deletions pw/patchwork.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,60 +29,17 @@ class PatchworkPostException(Exception):
pass


class PatchworkFetchException(Exception):
pass


def series_patches_ordered(series):
"""Return the patches of a series in the order they should be applied

Patchwork lists them in arrival order, use the n/total counter it parsed
out of the subject to put them back into the order the author intended.
"""
patches = series['patches']
total = series['total']
if total != len(patches):
core.log("Patch order - count does not add up?!", "")
return patches

ordered = list(patches)
for i in range(total):
found = False
name = patches[i]['name']
for j in range(total):
# scanning PW-parsed name - tags are separated by commas
if name.find(f" {j + 1}/{total}") >= 0 or \
name.find(f",{j + 1}/{total}") >= 0 or \
name.find(f"[{j + 1}/{total}") >= 0 or \
name.find(f"0{j + 1}/{total}") >= 0:
if ordered[j] is not patches[i]:
core.log(f"Patch order - reordering {i} => {j + 1}")
ordered[j] = patches[i]
found = True
break
if not found:
core.log("Patch order - not all patches were found!", "")
return patches
return ordered


class Patchwork(object):
# Patchwork mbox object types vs the names of the REST collections
_mbox_apis = {'cover': 'covers', 'patch': 'patches'}

def __init__(self, config):
self._session = requests.Session()
allowed_methods = Retry.DEFAULT_ALLOWED_METHODS | {'POST', 'PATCH'}
retry = Retry(connect=10, status=10,
status_forcelist={404, 429, 502, 503, 504},
retry = Retry(connect=10, status=10, status_forcelist={502, 504},
allowed_methods=allowed_methods, backoff_factor=1)
adapter = HTTPAdapter(max_retries=retry)
self._session.mount('http://', adapter)
self._session.mount('https://', adapter)

self.server = config.get('patchwork', 'server')
self.archive = config.get('patchwork', 'archive',
fallback='https://lore.kernel.org/all').rstrip('/')
ssl = config.getboolean('patchwork', 'use_ssl', fallback=True)
self._proto = "https://" if ssl else "http://"
self._token = config.get('patchwork', 'token', fallback='')
Expand Down Expand Up @@ -113,7 +70,7 @@ def _request(self, url):
try:
core.log("Response data", ret.json())
except json.decoder.JSONDecodeError:
core.log("Response data", ret.content.decode('utf-8', 'replace'))
core.log("Response data", ret.content.decode())
finally:
end = datetime.datetime.now()
core.log("Response time GET (sec)", (end - start).total_seconds())
Expand Down Expand Up @@ -179,30 +136,12 @@ def get_by_msgid(self, object_type, msgid):
msgid = urllib.parse.quote(msgid)
return self._get(f'{object_type}/?msgid={msgid}&project={self._project}', api='').json()

# Patchwork's own /mbox/ endpoints have been serving empty responses ever
# since one of its upgrades, so the messages come from the list archive.
# Patchwork is only asked for the message ids. Note that the archive
# requires a well-known user-agent, see the 'user-agent' config option.
def get_mbox_by_msgid(self, msgid):
url = f'{self.archive}/{urllib.parse.quote(msgid.strip("<>"))}/raw'
ret = self._request(url)
if ret.status_code != 200:
raise PatchworkFetchException(url, ret)
# Archives serve the message as it was posted, which is not necessarily
# valid UTF-8. Losing a character beats blowing up the entire series.
return ret.content.decode('utf-8', 'replace')

# Like patchwork's series mbox this contains the patches only, the cover
# letter is not part of it.
def series_to_mbox(self, series):
return ''.join([self.get_mbox_by_msgid(p['msgid'])
for p in series_patches_ordered(series)])
def get_mbox_direct(self, url):
return self._request(url).content.decode()

def get_mbox(self, object_type, identifier):
if object_type == 'series':
return self.series_to_mbox(self.get('series', identifier))
obj = self.get(self._mbox_apis[object_type], identifier)
return self.get_mbox_by_msgid(obj['msgid'])
url = f'{self._proto}{self.server}/{object_type}/{identifier}/mbox/'
return self._request(url).content.decode()

def _get(self, req, api='1.1'):
if api:
Expand Down
41 changes: 35 additions & 6 deletions pw/pw_series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from core import Series
from core import Patch
from core import log, log_open_sec, log_end_sec
from .patchwork import series_patches_ordered

# TODO: document

Expand All @@ -22,7 +21,7 @@ def __init__(self, pw, pw_series):
self.pull_url = None

if pw_series['cover_letter']:
pw_cover_letter = pw.get_mbox_by_msgid(pw_series['cover_letter']['msgid'])
pw_cover_letter = pw.get_mbox('cover', pw_series['cover_letter']['id'])
self.set_cover_letter(pw_cover_letter)
elif self.pw_series['patches']:
self.subject = self.pw_series['patches'][0]['name']
Expand All @@ -36,14 +35,44 @@ def __init__(self, pw, pw_series):
# Fast path incomplete series
if not pw_series['received_all']:
for p in self.pw_series['patches']:
raw_patch = pw.get_mbox_by_msgid(p['msgid'])
raw_patch = pw.get_mbox('patch', p['id'])
self.patches.append(Patch(raw_patch, p['id']))
return

# Do more magic around series which are complete
for p in series_patches_ordered(self.pw_series):
raw_patch = pw.get_mbox_by_msgid(p['msgid'])
self.add_patch(Patch(raw_patch, p['id']))
# Patchwork 2.2.2 orders them by arrival time
pids = []
for p in self.pw_series['patches']:
pids.append(p['id'])
total = self.pw_series['total']
if total == len(self.pw_series['patches']):
for i in range(total):
found = False
name = self.pw_series['patches'][i]['name']
pid = self.pw_series['patches'][i]['id']
for j in range(total):
# scanning PW-parsed name - tags are separated by commas
if name.find(f" {j + 1}/{total}") >= 0 or \
name.find(f",{j + 1}/{total}") >= 0 or \
name.find(f"[{j + 1}/{total}") >= 0 or \
name.find(f"0{j + 1}/{total}") >= 0:
if pids[j] != pid:
log(f"Patch order - reordering {i} => {j + 1}")
pids[j] = pid
found = True
break
if not found:
log("Patch order - not all patches were found!", "")
pids = []
for p in self.pw_series['patches']:
pids.append(p['id'])
break
else:
log("Patch order - count does not add up?!", "")

for pid in pids:
raw_patch = pw.get_mbox('patch', pid)
self.add_patch(Patch(raw_patch, pid))

if not pw_series['cover_letter']:
if len(self.patches) == 1:
Expand Down
3 changes: 2 additions & 1 deletion pw_brancher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ def apply_pending_patches(pw, config, tree, branch_name) -> Tuple[List, List]:
else:
log_open_sec("Applying: " + entry["series"][0]["name"])
seen_series.add(series_id)
data = pw.get_mbox('series', series_id)
mbox_url = entry["series"][0]["mbox"]
data = pw.get_mbox_direct(mbox_url)
p = Patch(data)
try:
tree.apply(p)
Expand Down