|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# A tool to add the key FirstOrbitMC to the MCProdInfo objects of an existing MC production. |
| 4 | +# FirstOrbitMC is the first orbit the MC used; analysis (AggregatedRunInfo::adjust_from_MC) |
| 5 | +# aligns its timeframes with it. |
| 6 | +# |
| 7 | +# The value is read per run from the log archives of the run's own GRID jobs, since a production |
| 8 | +# may contain runs made with different software. Runs without a reachable log archive can be |
| 9 | +# filled, and others cross-checked, by rerunning the anchoring step with the run's software. |
| 10 | +# |
| 11 | +# 1. scan (needs alien.py and a GRID token): read two log archives per run |
| 12 | +# mcprodinfo_firstorbit.py scan --tag LHC26j17 -o LHC26j17.json |
| 13 | +# 2. rerun (needs CVMFS): fill runs without archive and cross-check a sample |
| 14 | +# mcprodinfo_firstorbit.py rerun LHC26j17.json --crosscheck 3 |
| 15 | +# 3. apply (needs a GRID token with the right role): publish new MCProdInfo objects |
| 16 | +# mcprodinfo_firstorbit.py apply LHC26j17.json --comment "FirstOrbitMC added because of O2-7335" --yes |
| 17 | +# The apply step is a dry run unless --yes is given. |
| 18 | + |
| 19 | +import argparse |
| 20 | +import collections |
| 21 | +import concurrent.futures |
| 22 | +import json |
| 23 | +import os |
| 24 | +import random |
| 25 | +import re |
| 26 | +import subprocess |
| 27 | +import sys |
| 28 | +import tempfile |
| 29 | +import zipfile |
| 30 | +from urllib.parse import parse_qs, quote, urlparse |
| 31 | + |
| 32 | +import requests |
| 33 | +import urllib3 |
| 34 | + |
| 35 | +from mcprodinfo_ccdb_upload import MCProdInfo, publish_MCProdInfo |
| 36 | + |
| 37 | +MONALISA = "https://alimonitor.cern.ch" |
| 38 | + |
| 39 | + |
| 40 | +def token_cert(): |
| 41 | + key = os.environ.get("JALIEN_TOKEN_KEY") |
| 42 | + cert = os.environ.get("JALIEN_TOKEN_CERT") |
| 43 | + if key is None and cert is None: |
| 44 | + uid = os.getuid() |
| 45 | + cert, key = f"/tmp/tokencert_{uid}.pem", f"/tmp/tokenkey_{uid}.pem" |
| 46 | + return cert, key |
| 47 | + |
| 48 | + |
| 49 | +def list_mcprodinfo(ccdb_url, username, tag): |
| 50 | + """Newest MCProdInfo metadata per run of a production.""" |
| 51 | + url = f"{ccdb_url}/browse/Users/{username[0]}/{username}/MCProdInfo/{tag}" |
| 52 | + r = requests.get(url, headers={"Accept": "application/json"}, timeout=120) |
| 53 | + r.raise_for_status() |
| 54 | + newest = {} |
| 55 | + for o in r.json().get("objects", []): |
| 56 | + run = int(o["RunNumber"]) |
| 57 | + if run not in newest or int(o["createTime"]) > int(newest[run]["createTime"]): |
| 58 | + newest[run] = o |
| 59 | + return newest |
| 60 | + |
| 61 | + |
| 62 | +def newest_mcprodinfo(ccdb_url, username, tag, run): |
| 63 | + """Newest MCProdInfo object of one run, or None.""" |
| 64 | + url = f"{ccdb_url}/browse/Users/{username[0]}/{username}/MCProdInfo/{tag}/{run}/{run + 1}" |
| 65 | + r = requests.get(url, headers={"Accept": "application/json"}, cert=token_cert(), verify=False, timeout=120) |
| 66 | + r.raise_for_status() |
| 67 | + objs = r.json().get("objects", []) |
| 68 | + return max(objs, key=lambda o: int(o["createTime"])) if objs else None |
| 69 | + |
| 70 | + |
| 71 | +def masterjobs(tag): |
| 72 | + """Masterjob ids and output directories of a production, per run (from MonALISA).""" |
| 73 | + cert = token_cert() |
| 74 | + r = requests.get(f"{MONALISA}/MC/prodDetails.jsp?tag={quote(tag)}", cert=cert, verify=False, |
| 75 | + allow_redirects=False, timeout=120) |
| 76 | + jobtype = parse_qs(urlparse(r.headers.get("Location", "")).query).get("filter_jobtype", [None])[0] |
| 77 | + if not jobtype: |
| 78 | + raise RuntimeError(f"no MonALISA jobtype for production {tag}") |
| 79 | + r = requests.get(f"{MONALISA}/job_events.jsp?timesel=0&owner=aliprod&filter_jobtype={quote(jobtype)}&res_path=json", |
| 80 | + cert=cert, verify=False, timeout=300) |
| 81 | + jobs = collections.defaultdict(dict) |
| 82 | + for mjid, entry in r.json().items(): |
| 83 | + run, outdir = entry.get("run_no"), entry.get("outputdir") |
| 84 | + if str(run).isdigit() and outdir: |
| 85 | + jobs[int(run)][str(mjid)] = outdir.rstrip("/") |
| 86 | + return jobs |
| 87 | + |
| 88 | + |
| 89 | +def masterjob_package(mjid): |
| 90 | + """Packages of a masterjob, from its JDL.""" |
| 91 | + m = re.search(r"Packages\s*=\s*\{([^}]*)\}", alien("ps", "-jdl", mjid).stdout) |
| 92 | + return ",".join(re.findall(r'"([^"]+)"', m.group(1))) if m else None |
| 93 | + |
| 94 | + |
| 95 | +def alien(*args, timeout=300): |
| 96 | + return subprocess.run(["alien.py", *args], capture_output=True, text=True, timeout=timeout) |
| 97 | + |
| 98 | + |
| 99 | +def read_job(jobdir, run): |
| 100 | + """Package, FirstOrbitMC and data first orbit from the output of one job, or None. |
| 101 | +
|
| 102 | + The anchoring log and stdout are read from log_archive.zip, or from the job directory |
| 103 | + when a production stores them there as plain files.""" |
| 104 | + stdout, log = "", "" |
| 105 | + with tempfile.TemporaryDirectory() as d: |
| 106 | + local = os.path.join(d, "log_archive.zip") |
| 107 | + if alien("cp", f"alien://{jobdir}/log_archive.zip", f"file://{local}").returncode == 0: |
| 108 | + try: |
| 109 | + z = zipfile.ZipFile(local) |
| 110 | + stdout = z.read("stdout").decode(errors="ignore") if "stdout" in z.namelist() else "" |
| 111 | + ts = [n for n in z.namelist() if n.startswith("timestampsampling")] |
| 112 | + log = z.read(ts[0]).decode(errors="ignore") if ts else "" |
| 113 | + except (zipfile.BadZipFile, KeyError): |
| 114 | + pass |
| 115 | + for name in ("stdout", f"timestampsampling_{run}.log"): |
| 116 | + if (name == "stdout" and stdout) or (name != "stdout" and log): |
| 117 | + continue |
| 118 | + plain = os.path.join(d, name) |
| 119 | + if alien("cp", f"alien://{jobdir}/{name}", f"file://{plain}").returncode == 0: |
| 120 | + text = open(plain, errors="ignore").read() |
| 121 | + if name == "stdout": |
| 122 | + stdout = text |
| 123 | + else: |
| 124 | + log = text |
| 125 | + fo = re.search(r"Determined first orbit to be:\s+(\d+)", log) |
| 126 | + os_ = re.search(r"^OS\s+(\d+)", log, re.M) |
| 127 | + pkg = re.search(r"ALIEN_JDL_PACKAGES=(\S+)", stdout) |
| 128 | + if not (fo and os_): |
| 129 | + return None |
| 130 | + return {"lfn": jobdir, "package": pkg.group(1) if pkg else None, |
| 131 | + "FirstOrbitMC": int(fo.group(1)), "FirstOrbitData": int(os_.group(1))} |
| 132 | + |
| 133 | + |
| 134 | +def scan_run(run, outdirs, nsamples): |
| 135 | + """Reads up to nsamples log archives of a run, spread over its masterjobs.""" |
| 136 | + candidates = [] |
| 137 | + for outdir in sorted(outdirs): |
| 138 | + splits = sorted(s.strip("/") for s in alien("ls", outdir).stdout.split() if re.fullmatch(r"\d{3}/?", s)) |
| 139 | + if splits: |
| 140 | + candidates.append([f"{outdir}/{s}" for s in (splits[:2] + splits[-2:])]) |
| 141 | + samples = [] |
| 142 | + # round robin over the masterjobs, so that different masterjobs of one run are compared |
| 143 | + for i in range(4): |
| 144 | + for c in candidates: |
| 145 | + if len(samples) >= nsamples: |
| 146 | + break |
| 147 | + if i < len(c): |
| 148 | + s = read_job(c[i], run) |
| 149 | + if s: |
| 150 | + samples.append(s) |
| 151 | + return samples, bool(candidates) |
| 152 | + |
| 153 | + |
| 154 | +def scan(args): |
| 155 | + meta = list_mcprodinfo(args.ccdb, args.username, args.tag) |
| 156 | + jobs = masterjobs(args.tag) |
| 157 | + runs = sorted(r for r in meta if not args.runs or r in args.runs) |
| 158 | + rows = {} |
| 159 | + |
| 160 | + def one(run): |
| 161 | + mjs = jobs.get(run, {}) |
| 162 | + packages = {mj: masterjob_package(mj) for mj in mjs} |
| 163 | + return run, mjs, packages, scan_run(run, mjs.values(), args.samples) |
| 164 | + |
| 165 | + with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex: |
| 166 | + for run, mjs, mjpackages, (samples, has_output) in ex.map(one, runs): |
| 167 | + optf = int(meta[run]["OrbitsPerTF"]) |
| 168 | + row = {"run": run, "OrbitsPerTF": optf, "masterjobs": mjpackages, "samples": samples, "meta": meta[run]} |
| 169 | + firsts = {(s["FirstOrbitMC"], s["FirstOrbitData"]) for s in samples} |
| 170 | + packages = {p for p in mjpackages.values() if p} | {s["package"] for s in samples if s["package"]} |
| 171 | + if len(packages) == 1: |
| 172 | + row["package"] = next(iter(packages)) |
| 173 | + if not mjs: |
| 174 | + row["status"] = "NO_JOBS" |
| 175 | + elif not has_output: |
| 176 | + row["status"] = "NO_OUTPUT" |
| 177 | + elif len(packages) > 1 or len(firsts) > 1: |
| 178 | + row["status"] = "MIXED" |
| 179 | + elif not samples: |
| 180 | + row["status"] = "NO_ARCHIVE" |
| 181 | + else: |
| 182 | + (row["FirstOrbitMC"], row["FirstOrbitData"]), = firsts |
| 183 | + row["shift"] = (row["FirstOrbitMC"] - row["FirstOrbitData"]) % optf |
| 184 | + row["status"] = "OK" |
| 185 | + rows[run] = row |
| 186 | + print(f"{run} OrbitsPerTF={optf} {row['status']} samples={len(samples)} " |
| 187 | + f"package={row.get('package')} FirstOrbitMC={row.get('FirstOrbitMC')} shift={row.get('shift')}", flush=True) |
| 188 | + with open(args.output, "w") as f: |
| 189 | + json.dump({"tag": args.tag, "username": args.username, "rows": [rows[r] for r in runs]}, f, indent=1) |
| 190 | + status = collections.Counter(r["status"] for r in rows.values()) |
| 191 | + print(f"{args.output}: {dict(status)}") |
| 192 | + return 0 |
| 193 | + |
| 194 | + |
| 195 | +def rerun_anchoring(package, run, orbits_per_tf, tag, timeout=900): |
| 196 | + """Runs the anchoring step for one run; returns (data first orbit, MC first orbit).""" |
| 197 | + pkg = package.split("::")[-1] |
| 198 | + with tempfile.TemporaryDirectory(prefix=f"anchor_{run}_") as wd: |
| 199 | + inner = (f"python3 $O2DPG_ROOT/MC/bin/o2dpg_sim_workflow_anchored.py --run-number {run} --split-id 1 " |
| 200 | + f"--prod-split 100 --cycle 0 -tf 1 --orbitsPerTF {orbits_per_tf} -- -gen pythia8 -proc inel " |
| 201 | + f"-productionTag {tag} -e TGeant4 -j 8 -ns 1") |
| 202 | + cmd = ["/cvmfs/alice.cern.ch/bin/alienv", "setenv", f"O2PDPSuite/{pkg}", "-c", "bash", "-c", inner] |
| 203 | + env = {k: v for k, v in os.environ.items() if not k.startswith("ALIEN_JDL")} |
| 204 | + p = subprocess.run(cmd, cwd=wd, env=env, capture_output=True, text=True, timeout=timeout) |
| 205 | + out = p.stdout + p.stderr |
| 206 | + os_ = re.search(r"^OS\s+(\d+)", out, re.M) |
| 207 | + fo = re.search(r"Determined first orbit to be:\s+(\d+)", out) |
| 208 | + if not (os_ and fo): |
| 209 | + raise RuntimeError(f"run {run}: anchoring output not understood (rc {p.returncode}): {out[-300:]}") |
| 210 | + return int(os_.group(1)), int(fo.group(1)) |
| 211 | + |
| 212 | + |
| 213 | +def rerun(args): |
| 214 | + with open(args.table) as f: |
| 215 | + table = json.load(f) |
| 216 | + rows = table["rows"] |
| 217 | + todo = [r for r in rows if r["status"] == "NO_ARCHIVE"] |
| 218 | + ok = [r for r in rows if r["status"] == "OK" and r.get("package")] |
| 219 | + todo += random.Random(1).sample(ok, min(args.crosscheck, len(ok))) |
| 220 | + |
| 221 | + def one(row): |
| 222 | + package = args.package or row.get("package") |
| 223 | + if not package: |
| 224 | + return row, None, "no package known" |
| 225 | + try: |
| 226 | + return row, rerun_anchoring(package, row["run"], row["OrbitsPerTF"], table["tag"]), package |
| 227 | + except Exception as e: |
| 228 | + return row, None, str(e) |
| 229 | + |
| 230 | + with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex: |
| 231 | + for row, res, info in ex.map(one, todo): |
| 232 | + if res is None: |
| 233 | + print(f"{row['run']} rerun failed: {info}", flush=True) |
| 234 | + continue |
| 235 | + data_first, mc_first = res |
| 236 | + if row["status"] == "OK": |
| 237 | + agree = (mc_first, data_first) == (row["FirstOrbitMC"], row["FirstOrbitData"]) |
| 238 | + row["crosscheck"] = "agree" if agree else f"DIFFER rerun {mc_first}/{data_first}" |
| 239 | + print(f"{row['run']} cross-check {row['crosscheck']}", flush=True) |
| 240 | + else: |
| 241 | + row.update(status="RERUN", package=info, FirstOrbitMC=mc_first, FirstOrbitData=data_first, |
| 242 | + shift=(mc_first - data_first) % row["OrbitsPerTF"]) |
| 243 | + print(f"{row['run']} RERUN package={info} FirstOrbitMC={mc_first} shift={row['shift']}", flush=True) |
| 244 | + with open(args.table, "w") as f: |
| 245 | + json.dump(table, f, indent=1) |
| 246 | + return 0 |
| 247 | + |
| 248 | + |
| 249 | +def apply(args): |
| 250 | + with open(args.table) as f: |
| 251 | + table = json.load(f) |
| 252 | + tag = table["tag"] |
| 253 | + username = args.username or table["username"] |
| 254 | + for row in table["rows"]: |
| 255 | + if row["status"] not in ("OK", "RERUN") or str(row.get("crosscheck", "agree")) != "agree": |
| 256 | + print(f"SKIP {tag} run {row['run']} status={row['status']} crosscheck={row.get('crosscheck')}") |
| 257 | + continue |
| 258 | + m = row["meta"] |
| 259 | + # the newest object must still be the one scanned; skip runs that already carry the value |
| 260 | + current = newest_mcprodinfo(args.ccdb, username, tag, row["run"]) |
| 261 | + if current is not None and current.get("id") != m.get("id"): |
| 262 | + if str(current.get("FirstOrbitMC")) == str(row["FirstOrbitMC"]): |
| 263 | + print(f"DONE {tag} run {row['run']} already has FirstOrbitMC={row['FirstOrbitMC']}") |
| 264 | + continue |
| 265 | + print(f"ERROR: {tag} run {row['run']} has a newer object ({current.get('id')}) than the scanned one " |
| 266 | + f"({m.get('id')}), with FirstOrbitMC={current.get('FirstOrbitMC')}; scan again") |
| 267 | + return 1 |
| 268 | + info = MCProdInfo(LPMProductionTag=tag, |
| 269 | + Col=m.get("Col"), |
| 270 | + IntRate=float(m["IntRate"]) if m.get("IntRate") not in (None, "None") else None, |
| 271 | + RunNumber=row["run"], |
| 272 | + OrbitsPerTF=row["OrbitsPerTF"], |
| 273 | + Comment=args.comment, |
| 274 | + McTag=m.get("McTag") if m.get("McTag") not in (None, "None") else row.get("package"), |
| 275 | + RecoTag=m.get("RecoTag") if m.get("RecoTag") != "None" else None, |
| 276 | + FirstOrbitMC=row["FirstOrbitMC"]) |
| 277 | + print(f"{'PUBLISH' if args.yes else 'DRY'} {tag} run {row['run']} FirstOrbitMC={row['FirstOrbitMC']} " |
| 278 | + f"shift={row['shift']} McTag={info.McTag} -> {args.ccdb} Users/{username[0]}/{username}") |
| 279 | + if args.yes: |
| 280 | + r = publish_MCProdInfo(info, ccdb_url=args.ccdb, username=username, force_overwrite=True) |
| 281 | + where = r.headers.get("Location") or r.headers.get("Content-Location") if r is not None else None |
| 282 | + with open(args.log, "a") as log: |
| 283 | + log.write(f"{tag} {row['run']} {r.status_code if r is not None else None} {where}\n") |
| 284 | + if r is None or r.status_code not in (200, 201): |
| 285 | + print(f"ERROR: upload of run {row['run']} failed: {r.status_code if r is not None else 'no response'} " |
| 286 | + f"{(r.text[:300] if r is not None else '')}") |
| 287 | + return 1 |
| 288 | + return 0 |
| 289 | + |
| 290 | + |
| 291 | +def main(): |
| 292 | + urllib3.disable_warnings() |
| 293 | + parser = argparse.ArgumentParser(description="Add FirstOrbitMC to the MCProdInfo objects of an MC production") |
| 294 | + sub = parser.add_subparsers(dest="cmd", required=True) |
| 295 | + s = sub.add_parser("scan", help="read FirstOrbitMC per run from the log archives of its jobs") |
| 296 | + s.add_argument("--tag", required=True, help="LPM production tag") |
| 297 | + s.add_argument("--runs", type=int, nargs="*", help="restrict to these runs") |
| 298 | + s.add_argument("--samples", type=int, default=2, help="log archives to read per run") |
| 299 | + s.add_argument("--ccdb", default="http://alice-ccdb.cern.ch", help="CCDB to read MCProdInfo from") |
| 300 | + s.add_argument("--username", default="aliprod", help="owner of the MCProdInfo objects") |
| 301 | + s.add_argument("-j", "--jobs", type=int, default=8, help="runs processed in parallel") |
| 302 | + s.add_argument("-o", "--output", required=True, help="output table (JSON)") |
| 303 | + r = sub.add_parser("rerun", help="rerun the anchoring for runs without archive and for a cross-check sample") |
| 304 | + r.add_argument("table", help="table written by scan, updated in place") |
| 305 | + r.add_argument("--package", help="override the O2PDPSuite package of each run (default: from its masterjob JDL)") |
| 306 | + r.add_argument("--crosscheck", type=int, default=3, help="number of runs with archive to cross-check") |
| 307 | + r.add_argument("-j", "--jobs", type=int, default=8, help="parallel anchoring reruns") |
| 308 | + a = sub.add_parser("apply", help="publish MCProdInfo objects with FirstOrbitMC from a table") |
| 309 | + a.add_argument("table", help="table written by scan") |
| 310 | + a.add_argument("--comment", required=True, help="Comment stored in the new objects") |
| 311 | + a.add_argument("--ccdb", default="https://alice-ccdb.cern.ch", help="CCDB to publish to") |
| 312 | + a.add_argument("--username", help="owner path to publish under (default: as scanned)") |
| 313 | + a.add_argument("--yes", action="store_true", help="really publish (default: dry run)") |
| 314 | + a.add_argument("--log", default="mcprodinfo_firstorbit_published.log", help="appends tag, run, HTTP status and object location per upload") |
| 315 | + args = parser.parse_args() |
| 316 | + return {"scan": scan, "rerun": rerun, "apply": apply}[args.cmd](args) |
| 317 | + |
| 318 | + |
| 319 | +if __name__ == "__main__": |
| 320 | + sys.exit(main()) |
0 commit comments