From 948c812fceb1fe6bfa034fe24e79ffaa41d34a10 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Thu, 10 Sep 2026 18:27:43 +0200 Subject: [PATCH 1/6] contest: pw: config to change PW's check name To prepare a split between HW and the rest. Signed-off-by: Matthieu Baerts --- pw_contest.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pw_contest.py b/pw_contest.py index f118bb1..1f1b2c3 100755 --- a/pw_contest.py +++ b/pw_contest.py @@ -27,6 +27,8 @@ patch_state=state.json [www] contest=https://server-with-ui/contest.html +[patchwork] +check_name=contest """ class Codes: @@ -223,16 +225,16 @@ def skip_update(outcome) -> bool: return False -def update_one(pw, patch_id, outcome, link): +def update_one(pw, patch_id, outcome, link, check_name): description = outcome['branch'] if outcome["code"] >= 0: description += f' (tests: {outcome["cnt"]})' url = link + '?pw-n=0&branch=' + outcome['branch'] - pw.post_check(patch_id, name="contest", state=code_to_pw[outcome["code"]], + pw.post_check(patch_id, name=check_name, state=code_to_pw[outcome["code"]], url=url, desc=description) -def _patch_state_update(pw, state: dict, link: str): +def _patch_state_update(pw, state: dict, link: str, check_name: str): update_cnt = 0 for series_id, outcome in state["series"].items(): if skip_update(outcome): @@ -242,7 +244,7 @@ def _patch_state_update(pw, state: dict, link: str): log_open_sec('Updating series ' + series_id) series_pw = pw.get("series", series_id) for patch in series_pw["patches"]: - update_one(pw, patch["id"], outcome, link) + update_one(pw, patch["id"], outcome, link, check_name) update_cnt += 1 del outcome["update"] @@ -255,7 +257,7 @@ def _patch_state_update(pw, state: dict, link: str): try: log_open_sec('Updating PR ' + pr_id) - update_one(pw, pr_id, outcome, link) + update_one(pw, pr_id, outcome, link, check_name) update_cnt += 1 del outcome["update"] @@ -265,10 +267,10 @@ def _patch_state_update(pw, state: dict, link: str): print("Updated", update_cnt, "pw things") -def patch_state_update(pw, state: dict, link: str): +def patch_state_update(pw, state: dict, link: str, check_name: str): log_open_sec('Updating patch states') try: - _patch_state_update(pw, state, link) + _patch_state_update(pw, state, link, check_name) finally: log_end_sec() @@ -291,7 +293,8 @@ def main_loop(pw) -> int: results_by_branch = results_pivot(filters, results) branch_outcome = branch_summarize(filters, results_by_branch) patch_state_compute(patch_state, branches, branch_outcome) - patch_state_update(pw, patch_state, config.get('www', 'contest')) + patch_state_update(pw, patch_state, config.get('www', 'contest'), + config.get('patchwork', 'check_name', fallback='contest')) rbb = config.get('output', 'results_by_branch', fallback=None) if rbb: From d8438de501fec73576c284eab10f241468a43277 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Thu, 10 Sep 2026 19:03:41 +0200 Subject: [PATCH 2/6] brancher: write info file atomically This file is read by another service (faker), possibly simultaneously. Signed-off-by: Matthieu Baerts --- pw_brancher.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pw_brancher.py b/pw_brancher.py index b2013e6..5a544dd 100755 --- a/pw_brancher.py +++ b/pw_brancher.py @@ -390,10 +390,7 @@ def dump_branches(config, state) -> None: "url": pub_url + " " + name}) write_json_atomic(config.get("output", "branches"), data) - - info = config.get("output", "info") - with open(info, 'w') as fp: - json.dump(state["info"], fp) + write_json_atomic(config.get("output", "info"), state["info"]) log_end_sec() From 349a4c35ed0eca5933d10113492920586ebb25cc Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Fri, 11 Sep 2026 15:33:48 +0200 Subject: [PATCH 3/6] contest: faker: drop branches support It looks like it is not / no longer used. So drop that. Note that the combined 'info' file will now only be used by the UI (status page), not by pw_contest.py anymore. Signed-off-by: Matthieu Baerts --- contest/results-faker.py | 53 ---------------------------------------- 1 file changed, 53 deletions(-) diff --git a/contest/results-faker.py b/contest/results-faker.py index d2c6528..8e927e6 100755 --- a/contest/results-faker.py +++ b/contest/results-faker.py @@ -15,11 +15,8 @@ Config: [input] -branches=/path/to/branches.json,/path/to/branches2.json infos=/path/to/infos.json,/path/to/infos2.json [output] -dir=/path/to/output -url_pfx=relative/within/server info=/path/to/info.json """ @@ -47,56 +44,6 @@ def main() -> None: combine_infos(config) - branches = [] - paths = config.get("input", "branches") - for path in paths.split(','): - with open(path, "r") as fp: - branches += json.load(fp) - - branches = sorted(branches, key=lambda x: x["date"]) - - url = config.get("output", "url_pfx") - if url[-1] != '/': - url += '/' - directory = config.get("output", "dir") - - used_cookies = set() - results = [] - for br in branches: - br_dt = datetime.datetime.fromisoformat(br["date"]) - run_id_cookie = int(br_dt.timestamp() / 60) % 1000000 - while run_id_cookie in used_cookies: - run_id_cookie += 1 - used_cookies.add(run_id_cookie) - fname = f"results-{run_id_cookie}.json" - - data = {'url': url + fname, - 'branch': br["branch"], - 'executor': "brancher"} - results.append(data) - - run = {'branch': br["branch"], 'executor': "brancher"} - br_dt += datetime.timedelta(seconds=1) - run["start"] = br_dt.isoformat() - br_dt += datetime.timedelta(seconds=3) - run["end"] = br_dt.isoformat() - - tail = br["url"].find('.git ') - if br["url"].startswith('https://github.com') and tail > 0: - br_url = br["url"][:tail] + "/commits/" + br["url"][tail + 5:] - else: - br_url = "https://netdev.bots.linux.dev/contest/branches.json" - - run["results"] = [ - {"test": "branch-created", "group": "---", "result": "pass", "link": br_url} - ] - - with open(os.path.join(directory, fname), "w") as fp: - json.dump(run, fp) - - with open(os.path.join(directory, 'results.json'), "w") as fp: - json.dump(results, fp) - if __name__ == "__main__": main() From 240142ddd2953bb9b8ef2ad27717c135c8ccf420 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Fri, 11 Sep 2026 15:11:41 +0200 Subject: [PATCH 4/6] contest: collector: multiple input branches To support the HW branches, deal with multiple input branch URLs, and multiple output combined files. This sounds better than having to deal with different services writing in the same DB. Config changes: - branch_url -> branch_urls: with multiple URLs, separated by ',' - combined -> combined_files: output files, separated by ',' Signed-off-by: Matthieu Baerts --- contest/results-collector.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/contest/results-collector.py b/contest/results-collector.py index c1f30ed..eac884d 100755 --- a/contest/results-collector.py +++ b/contest/results-collector.py @@ -19,10 +19,11 @@ refresh=#secs [input] remote_db=/path/to/db +branch_urls=url1,url2 [output] dir=/path/to/output url_pfx=relative/within/server -combined=name-of-manifest.json +combined_files=name-of-manifest1.json,name-of-manifest2.json [db] db=db-name stability-name=table-name @@ -469,8 +470,8 @@ def filter_l1_l2(case): data["results"] = list(filter(lambda x: x is not None, data["results"])) -def build_combined(fetcher, remote_db): - r = requests.get(fetcher.config.get('input', 'branch_url')) +def build_combined(fetcher, remote_db, branch_url): + r = requests.get(branch_url) branches = json.loads(r.content.decode('utf-8')) branch_info = {} for br in branches: @@ -573,6 +574,13 @@ def build_seen(fetcher, remote_db): def main() -> None: fetcher = FetcherState() + branch_urls = fetcher.config.get('input', 'branch_urls').split(',') + combined_files = fetcher.config.get('output', 'combined_files').split(',') + if len(branch_urls) != len(combined_files): + raise ValueError( + "'branch_urls' and 'combined_files' must contain the same number " + "of comma-separated entries") + with open(fetcher.config.get('input', 'remote_db'), "r") as fp: remote_db = json.load(fp) @@ -586,10 +594,11 @@ def main() -> None: if fetcher.fetched: print('Generating combined') - results = build_combined(fetcher, remote_db) - combined = os.path.join(fetcher.config.get('output', 'combined')) - write_json_atomic(combined, results) + for branch_url, combined_file in zip(branch_urls, combined_files): + print('From', branch_url, 'to', combined_file) + results = build_combined(fetcher, remote_db, branch_url) + write_json_atomic(combined_file, results) time.sleep(int(fetcher.config.get('cfg', 'refresh'))) From 7f76dc7007b62c841a87424e61f06d7756150827 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Fri, 11 Sep 2026 16:05:00 +0200 Subject: [PATCH 5/6] contest: collector: skip results not linked to a recent branch This limitation was only done for in progress work, but we don't need the old results. Old results are still on the different runners (and in the DB). (Probably the different runners should also trim their 'results.json' file.) Because of that, 'all-results.json' is currently over 410MB, which takes a bit of time to process here and in pw_contest. The other consequence is that some files for the UI contains info about old branches that are not needed, but are still processed on the client side, e.g. branch-results.json. Signed-off-by: Matthieu Baerts --- contest/results-collector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contest/results-collector.py b/contest/results-collector.py index eac884d..eec7a38 100755 --- a/contest/results-collector.py +++ b/contest/results-collector.py @@ -495,9 +495,10 @@ def build_combined(fetcher, remote_db, branch_url): report_broken_remote(remote, error) for entry in results: + if entry['branch'] not in branch_info: + continue + if not entry['url']: # Executor is running - if entry['branch'] not in branch_info: - continue data = entry.copy() when = datetime.datetime.fromisoformat(branch_info[entry['branch']]['date']) data["start"] = str(when) From 558538d0eb18a1cbd04eafc2e83b163f3aa252d9 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Fri, 11 Sep 2026 16:55:20 +0200 Subject: [PATCH 6/6] status: open HW branch results Generated in another file by the dedicated pw-contest service. Signed-off-by: Matthieu Baerts --- scripts/ui_assets.sh | 1 + ui/status.js | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/ui_assets.sh b/scripts/ui_assets.sh index 90c1aa9..28318d3 100755 --- a/scripts/ui_assets.sh +++ b/scripts/ui_assets.sh @@ -19,6 +19,7 @@ ASSETS=( "status.json" "issues.json" "contest/branch-results.json" + "contest/branch-results-hw.json" "contest/branches-info.json" "contest/filters.json" "contest/all-results.json" diff --git a/ui/status.js b/ui/status.js index 0c2d672..903ff3a 100644 --- a/ui/status.js +++ b/ui/status.js @@ -1001,7 +1001,7 @@ function load_result_table(data_raw, reload) } } -let xfr_todo = 4; +let xfr_todo = 5; let all_results = null; let branches_info = null; let branches = new Set(); @@ -1272,6 +1272,9 @@ function do_it() $(document).ready(function() { $.get("contest/branch-results.json", branch_res_doit) }); + $(document).ready(function() { + $.get("contest/branch-results-hw.json", branch_res_doit) + }); $(document).ready(function() { $.get("query/results?branches=10&pending=y", results_loaded) });