diff --git a/contest/results-collector.py b/contest/results-collector.py index c1f30ed5..eec7a388 100755 --- a/contest/results-collector.py +++ b/contest/results-collector.py @@ -19,10 +19,11 @@ refresh=#secs [input] remote_db=/path/to/db +branch_urls=url1,url2 [output] dir=/path/to/output url_pfx=relative/within/server -combined=name-of-manifest.json +combined_files=name-of-manifest1.json,name-of-manifest2.json [db] db=db-name stability-name=table-name @@ -469,8 +470,8 @@ def filter_l1_l2(case): data["results"] = list(filter(lambda x: x is not None, data["results"])) -def build_combined(fetcher, remote_db): - r = requests.get(fetcher.config.get('input', 'branch_url')) +def build_combined(fetcher, remote_db, branch_url): + r = requests.get(branch_url) branches = json.loads(r.content.decode('utf-8')) branch_info = {} for br in branches: @@ -494,9 +495,10 @@ def build_combined(fetcher, remote_db): report_broken_remote(remote, error) for entry in results: + if entry['branch'] not in branch_info: + continue + if not entry['url']: # Executor is running - if entry['branch'] not in branch_info: - continue data = entry.copy() when = datetime.datetime.fromisoformat(branch_info[entry['branch']]['date']) data["start"] = str(when) @@ -573,6 +575,13 @@ def build_seen(fetcher, remote_db): def main() -> None: fetcher = FetcherState() + branch_urls = fetcher.config.get('input', 'branch_urls').split(',') + combined_files = fetcher.config.get('output', 'combined_files').split(',') + if len(branch_urls) != len(combined_files): + raise ValueError( + "'branch_urls' and 'combined_files' must contain the same number " + "of comma-separated entries") + with open(fetcher.config.get('input', 'remote_db'), "r") as fp: remote_db = json.load(fp) @@ -586,10 +595,11 @@ def main() -> None: if fetcher.fetched: print('Generating combined') - results = build_combined(fetcher, remote_db) - combined = os.path.join(fetcher.config.get('output', 'combined')) - write_json_atomic(combined, results) + for branch_url, combined_file in zip(branch_urls, combined_files): + print('From', branch_url, 'to', combined_file) + results = build_combined(fetcher, remote_db, branch_url) + write_json_atomic(combined_file, results) time.sleep(int(fetcher.config.get('cfg', 'refresh'))) diff --git a/contest/results-faker.py b/contest/results-faker.py index d2c6528f..8e927e6a 100755 --- a/contest/results-faker.py +++ b/contest/results-faker.py @@ -15,11 +15,8 @@ Config: [input] -branches=/path/to/branches.json,/path/to/branches2.json infos=/path/to/infos.json,/path/to/infos2.json [output] -dir=/path/to/output -url_pfx=relative/within/server info=/path/to/info.json """ @@ -47,56 +44,6 @@ def main() -> None: combine_infos(config) - branches = [] - paths = config.get("input", "branches") - for path in paths.split(','): - with open(path, "r") as fp: - branches += json.load(fp) - - branches = sorted(branches, key=lambda x: x["date"]) - - url = config.get("output", "url_pfx") - if url[-1] != '/': - url += '/' - directory = config.get("output", "dir") - - used_cookies = set() - results = [] - for br in branches: - br_dt = datetime.datetime.fromisoformat(br["date"]) - run_id_cookie = int(br_dt.timestamp() / 60) % 1000000 - while run_id_cookie in used_cookies: - run_id_cookie += 1 - used_cookies.add(run_id_cookie) - fname = f"results-{run_id_cookie}.json" - - data = {'url': url + fname, - 'branch': br["branch"], - 'executor': "brancher"} - results.append(data) - - run = {'branch': br["branch"], 'executor': "brancher"} - br_dt += datetime.timedelta(seconds=1) - run["start"] = br_dt.isoformat() - br_dt += datetime.timedelta(seconds=3) - run["end"] = br_dt.isoformat() - - tail = br["url"].find('.git ') - if br["url"].startswith('https://github.com') and tail > 0: - br_url = br["url"][:tail] + "/commits/" + br["url"][tail + 5:] - else: - br_url = "https://netdev.bots.linux.dev/contest/branches.json" - - run["results"] = [ - {"test": "branch-created", "group": "---", "result": "pass", "link": br_url} - ] - - with open(os.path.join(directory, fname), "w") as fp: - json.dump(run, fp) - - with open(os.path.join(directory, 'results.json'), "w") as fp: - json.dump(results, fp) - if __name__ == "__main__": main() diff --git a/pw_brancher.py b/pw_brancher.py index b2013e62..5a544dd9 100755 --- a/pw_brancher.py +++ b/pw_brancher.py @@ -390,10 +390,7 @@ def dump_branches(config, state) -> None: "url": pub_url + " " + name}) write_json_atomic(config.get("output", "branches"), data) - - info = config.get("output", "info") - with open(info, 'w') as fp: - json.dump(state["info"], fp) + write_json_atomic(config.get("output", "info"), state["info"]) log_end_sec() diff --git a/pw_contest.py b/pw_contest.py index f118bb15..1f1b2c36 100755 --- a/pw_contest.py +++ b/pw_contest.py @@ -27,6 +27,8 @@ patch_state=state.json [www] contest=https://server-with-ui/contest.html +[patchwork] +check_name=contest """ class Codes: @@ -223,16 +225,16 @@ def skip_update(outcome) -> bool: return False -def update_one(pw, patch_id, outcome, link): +def update_one(pw, patch_id, outcome, link, check_name): description = outcome['branch'] if outcome["code"] >= 0: description += f' (tests: {outcome["cnt"]})' url = link + '?pw-n=0&branch=' + outcome['branch'] - pw.post_check(patch_id, name="contest", state=code_to_pw[outcome["code"]], + pw.post_check(patch_id, name=check_name, state=code_to_pw[outcome["code"]], url=url, desc=description) -def _patch_state_update(pw, state: dict, link: str): +def _patch_state_update(pw, state: dict, link: str, check_name: str): update_cnt = 0 for series_id, outcome in state["series"].items(): if skip_update(outcome): @@ -242,7 +244,7 @@ def _patch_state_update(pw, state: dict, link: str): log_open_sec('Updating series ' + series_id) series_pw = pw.get("series", series_id) for patch in series_pw["patches"]: - update_one(pw, patch["id"], outcome, link) + update_one(pw, patch["id"], outcome, link, check_name) update_cnt += 1 del outcome["update"] @@ -255,7 +257,7 @@ def _patch_state_update(pw, state: dict, link: str): try: log_open_sec('Updating PR ' + pr_id) - update_one(pw, pr_id, outcome, link) + update_one(pw, pr_id, outcome, link, check_name) update_cnt += 1 del outcome["update"] @@ -265,10 +267,10 @@ def _patch_state_update(pw, state: dict, link: str): print("Updated", update_cnt, "pw things") -def patch_state_update(pw, state: dict, link: str): +def patch_state_update(pw, state: dict, link: str, check_name: str): log_open_sec('Updating patch states') try: - _patch_state_update(pw, state, link) + _patch_state_update(pw, state, link, check_name) finally: log_end_sec() @@ -291,7 +293,8 @@ def main_loop(pw) -> int: results_by_branch = results_pivot(filters, results) branch_outcome = branch_summarize(filters, results_by_branch) patch_state_compute(patch_state, branches, branch_outcome) - patch_state_update(pw, patch_state, config.get('www', 'contest')) + patch_state_update(pw, patch_state, config.get('www', 'contest'), + config.get('patchwork', 'check_name', fallback='contest')) rbb = config.get('output', 'results_by_branch', fallback=None) if rbb: diff --git a/scripts/ui_assets.sh b/scripts/ui_assets.sh index 90c1aa9e..28318d3b 100755 --- a/scripts/ui_assets.sh +++ b/scripts/ui_assets.sh @@ -19,6 +19,7 @@ ASSETS=( "status.json" "issues.json" "contest/branch-results.json" + "contest/branch-results-hw.json" "contest/branches-info.json" "contest/filters.json" "contest/all-results.json" diff --git a/ui/status.js b/ui/status.js index 0c2d672e..903ff3a4 100644 --- a/ui/status.js +++ b/ui/status.js @@ -1001,7 +1001,7 @@ function load_result_table(data_raw, reload) } } -let xfr_todo = 4; +let xfr_todo = 5; let all_results = null; let branches_info = null; let branches = new Set(); @@ -1272,6 +1272,9 @@ function do_it() $(document).ready(function() { $.get("contest/branch-results.json", branch_res_doit) }); + $(document).ready(function() { + $.get("contest/branch-results-hw.json", branch_res_doit) + }); $(document).ready(function() { $.get("query/results?branches=10&pending=y", results_loaded) });