Contest: prepare HW split

contest/results-collector.py

@@ -19,10 +19,11 @@
 refresh=#secs
 [input]
 remote_db=/path/to/db
+branch_urls=url1,url2
 [output]
 dir=/path/to/output
 url_pfx=relative/within/server
-combined=name-of-manifest.json
+combined_files=name-of-manifest1.json,name-of-manifest2.json
 [db]
 db=db-name
 stability-name=table-name
@@ -469,8 +470,8 @@ def filter_l1_l2(case):
     data["results"] = list(filter(lambda x: x is not None, data["results"]))
 
 
-def build_combined(fetcher, remote_db):
-    r = requests.get(fetcher.config.get('input', 'branch_url'))
+def build_combined(fetcher, remote_db, branch_url):
+    r = requests.get(branch_url)
     branches = json.loads(r.content.decode('utf-8'))
     branch_info = {}
     for br in branches:
@@ -494,9 +495,10 @@ def build_combined(fetcher, remote_db):
             report_broken_remote(remote, error)
 
         for entry in results:
+            if entry['branch'] not in branch_info:
+                continue
+
             if not entry['url']:    # Executor is running
-                if entry['branch'] not in branch_info:
-                    continue
                 data = entry.copy()
                 when = datetime.datetime.fromisoformat(branch_info[entry['branch']]['date'])
                 data["start"] = str(when)
@@ -573,6 +575,13 @@ def build_seen(fetcher, remote_db):
 def main() -> None:
     fetcher = FetcherState()
 
+    branch_urls = fetcher.config.get('input', 'branch_urls').split(',')
+    combined_files = fetcher.config.get('output', 'combined_files').split(',')
+    if len(branch_urls) != len(combined_files):
+        raise ValueError(
+            "'branch_urls' and 'combined_files' must contain the same number "
+            "of comma-separated entries")
+
     with open(fetcher.config.get('input', 'remote_db'), "r") as fp:
         remote_db = json.load(fp)
 
@@ -586,10 +595,11 @@ def main() -> None:
 
         if fetcher.fetched:
             print('Generating combined')
-            results = build_combined(fetcher, remote_db)
 
-            combined = os.path.join(fetcher.config.get('output', 'combined'))
-            write_json_atomic(combined, results)
+            for branch_url, combined_file in zip(branch_urls, combined_files):
+                print('From', branch_url, 'to', combined_file)
+                results = build_combined(fetcher, remote_db, branch_url)
+                write_json_atomic(combined_file, results)
 
         time.sleep(int(fetcher.config.get('cfg', 'refresh')))
 

contest/results-faker.py

@@ -15,11 +15,8 @@
 Config:
 
 [input]
-branches=/path/to/branches.json,/path/to/branches2.json
 infos=/path/to/infos.json,/path/to/infos2.json
 [output]
-dir=/path/to/output
-url_pfx=relative/within/server
 info=/path/to/info.json
 """
 
@@ -47,56 +44,6 @@ def main() -> None:
 
     combine_infos(config)
 
-    branches = []
-    paths = config.get("input", "branches")
-    for path in paths.split(','):
-        with open(path, "r") as fp:
-            branches += json.load(fp)
-
-    branches = sorted(branches, key=lambda x: x["date"])
-
-    url = config.get("output", "url_pfx")
-    if url[-1] != '/':
-        url += '/'
-    directory = config.get("output", "dir")
-
-    used_cookies = set()
-    results = []
-    for br in branches:
-        br_dt = datetime.datetime.fromisoformat(br["date"])
-        run_id_cookie = int(br_dt.timestamp() / 60) % 1000000
-        while run_id_cookie in used_cookies:
-            run_id_cookie += 1
-        used_cookies.add(run_id_cookie)
-        fname = f"results-{run_id_cookie}.json"
-
-        data = {'url': url + fname,
-               'branch': br["branch"],
-               'executor': "brancher"}
-        results.append(data)
-
-        run = {'branch': br["branch"], 'executor': "brancher"}
-        br_dt += datetime.timedelta(seconds=1)
-        run["start"] = br_dt.isoformat()
-        br_dt += datetime.timedelta(seconds=3)
-        run["end"] = br_dt.isoformat()
-
-        tail = br["url"].find('.git ')
-        if br["url"].startswith('https://github.com') and tail > 0:
-            br_url = br["url"][:tail] + "/commits/" + br["url"][tail + 5:]
-        else:
-            br_url = "https://netdev.bots.linux.dev/contest/branches.json"
-
-        run["results"] = [
-            {"test": "branch-created", "group": "---", "result": "pass", "link": br_url}
-        ]
-
-        with open(os.path.join(directory, fname), "w") as fp:
-            json.dump(run, fp)
-
-    with open(os.path.join(directory, 'results.json'), "w") as fp:
-        json.dump(results, fp)
-
 
 if __name__ == "__main__":
     main()

pw_brancher.py

@@ -390,10 +390,7 @@ def dump_branches(config, state) -> None:
                      "url": pub_url + " " + name})
 
     write_json_atomic(config.get("output", "branches"), data)
-
-    info = config.get("output", "info")
-    with open(info, 'w') as fp:
-        json.dump(state["info"], fp)
+    write_json_atomic(config.get("output", "info"), state["info"])
     log_end_sec()
 
 

pw_contest.py

@@ -27,6 +27,8 @@
 patch_state=state.json
 [www]
 contest=https://server-with-ui/contest.html
+[patchwork]
+check_name=contest
 """
 
 class Codes:
@@ -223,16 +225,16 @@ def skip_update(outcome) -> bool:
     return False
 
 
-def update_one(pw, patch_id, outcome, link):
+def update_one(pw, patch_id, outcome, link, check_name):
     description = outcome['branch']
     if outcome["code"] >= 0:
         description += f' (tests: {outcome["cnt"]})'
     url = link + '?pw-n=0&branch=' + outcome['branch']
-    pw.post_check(patch_id, name="contest", state=code_to_pw[outcome["code"]],
+    pw.post_check(patch_id, name=check_name, state=code_to_pw[outcome["code"]],
                   url=url, desc=description)
 
 
-def _patch_state_update(pw, state: dict, link: str):
+def _patch_state_update(pw, state: dict, link: str, check_name: str):
     update_cnt = 0
     for series_id, outcome in state["series"].items():
         if skip_update(outcome):
@@ -242,7 +244,7 @@ def _patch_state_update(pw, state: dict, link: str):
             log_open_sec('Updating series ' + series_id)
             series_pw = pw.get("series", series_id)
             for patch in series_pw["patches"]:
-                update_one(pw, patch["id"], outcome, link)
+                update_one(pw, patch["id"], outcome, link, check_name)
             update_cnt += 1
 
             del outcome["update"]
@@ -255,7 +257,7 @@ def _patch_state_update(pw, state: dict, link: str):
 
         try:
             log_open_sec('Updating PR ' + pr_id)
-            update_one(pw, pr_id, outcome, link)
+            update_one(pw, pr_id, outcome, link, check_name)
             update_cnt += 1
 
             del outcome["update"]
@@ -265,10 +267,10 @@ def _patch_state_update(pw, state: dict, link: str):
         print("Updated", update_cnt, "pw things")
 
 
-def patch_state_update(pw, state: dict, link: str):
+def patch_state_update(pw, state: dict, link: str, check_name: str):
     log_open_sec('Updating patch states')
     try:
-        _patch_state_update(pw, state, link)
+        _patch_state_update(pw, state, link, check_name)
     finally:
         log_end_sec()
 
@@ -291,7 +293,8 @@ def main_loop(pw) -> int:
     results_by_branch = results_pivot(filters, results)
     branch_outcome = branch_summarize(filters, results_by_branch)
     patch_state_compute(patch_state, branches, branch_outcome)
-    patch_state_update(pw, patch_state, config.get('www', 'contest'))
+    patch_state_update(pw, patch_state, config.get('www', 'contest'),
+                       config.get('patchwork', 'check_name', fallback='contest'))
 
     rbb = config.get('output', 'results_by_branch', fallback=None)
     if rbb:

scripts/ui_assets.sh

@@ -19,6 +19,7 @@ ASSETS=(
   "status.json"
   "issues.json"
   "contest/branch-results.json"
+  "contest/branch-results-hw.json"
   "contest/branches-info.json"
   "contest/filters.json"
   "contest/all-results.json"

ui/status.js

@@ -1001,7 +1001,7 @@ function load_result_table(data_raw, reload)
     }
 }
 
-let xfr_todo = 4;
+let xfr_todo = 5;
 let all_results = null;
 let branches_info = null;
 let branches = new Set();
@@ -1272,6 +1272,9 @@ function do_it()
     $(document).ready(function() {
         $.get("contest/branch-results.json", branch_res_doit)
     });
+    $(document).ready(function() {
+        $.get("contest/branch-results-hw.json", branch_res_doit)
+    });
     $(document).ready(function() {
         $.get("query/results?branches=10&pending=y", results_loaded)
     });