Skip to content

Commit 8954a02

Browse files
sawenzelclaude
andcommitted
Record FirstOrbitMC and the software version in MCProdInfo
This stores the first orbit used by an anchored MC job and its software version in its MCProdInfo object, and provides a new tool mcprodinfo_firstorbit.py that adds the first orbit to existing productions. - Analysis uses FirstOrbitMC to align its timeframes with the MC. - McTag is now filled at job time from ALIEN_JDL_PACKAGES. - The object is published again when the stored FirstOrbitMC differs. - A writer without FirstOrbitMC, e.g. the harvester with --overwrite, keeps the stored value. - Optional metadata with value None is no longer uploaded as the string "None". - Reading an object that lacks a required field no longer raises a TypeError. - The tool reads FirstOrbitMC per run from the log archives of the run's own jobs, found via MonALISA. - The software of each run is taken from its masterjob JDL and log archives; runs whose jobs disagree are flagged and skipped. - Runs without reachable archive are filled, and a sample cross-checked, by rerunning the anchoring with the run's software. - apply skips runs that already carry the value and stops if a run changed since the scan. Usage: mcprodinfo_firstorbit.py scan --tag LHC26h3 -o LHC26h3.json mcprodinfo_firstorbit.py rerun LHC26h3.json --crosscheck 3 mcprodinfo_firstorbit.py apply LHC26h3.json --comment "FirstOrbitMC added because of O2-7335" --yes For LHC26h3 the scan gives 60 runs made with MC-prod-2026-v12 (shift 2), 3 runs made with v14 (aligned) and 4 runs without jobs. https://its.cern.ch/jira/browse/O2-7335 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent ef75c8f commit 8954a02

3 files changed

Lines changed: 334 additions & 3 deletions

File tree

‎MC/bin/o2dpg_sim_workflow_anchored.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -811,6 +811,7 @@ def main():
811811
IntRate =rate,
812812
RunNumber = args.run_number,
813813
OrbitsPerTF = GLOparams["OrbitsPerTF"],
814+
McTag = os.getenv("ALIEN_JDL_PACKAGES"),
814815
FirstOrbitMC = effective_first_orbit)
815816
publish_MCProdInfo(info, username = grid_user_name, ccdb_url = mcprod_ccdb_server)
816817
else:

‎MC/prodinfo/mcprodinfo_ccdb_upload.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ class MCProdInfo:
2222
Comment: Optional[str] = None
2323
McTag: Optional[str] = None # main software tag used
2424
RecoTag: Optional[str] = None # RecoTag (if any)
25+
FirstOrbitMC: Optional[int] = None # first orbit used by the MC; its TFs start at FirstOrbitMC + k * OrbitsPerTF
2526
Hash: Optional[str] = field(default=None)
2627

2728
def __post_init__(self):
@@ -95,6 +96,9 @@ def filter_known_fields(cls, data: dict) -> dict:
9596
return {k: v for k, v in data.items() if k in valid_keys}
9697

9798
clean_meta = filter_known_fields(MCProdInfo, meta[0])
99+
# an object may lack fields that are required here (not every writer sets all of them)
100+
for f in fields(MCProdInfo):
101+
clean_meta.setdefault(f.name, None)
98102
return MCProdInfo(**clean_meta)
99103

100104
return None
@@ -127,7 +131,8 @@ def upload_mcprodinfo_meta(base_url, user, run_number, lpm_prod_tag, keys, cert_
127131
key_path = os.path.join(cert_dir, f"tokenkey_{uid}.pem")
128132

129133
# Build full URL
130-
query = "/".join(f"{k}={v}" for k, v in keys.items())
134+
optional = {f.name for f in fields(MCProdInfo) if f.default is not dataclasses.MISSING}
135+
query = "/".join(f"{k}={v}" for k, v in keys.items() if v is not None or k not in optional)
131136
user_path = 'Users/' + user[0] + '/' + user
132137
start = run_number
133138
stop = run_number + 1
@@ -154,7 +159,12 @@ def publish_MCProdInfo(mc_prod_info, ccdb_url = "https://alice-ccdb.cern.ch", us
154159

155160
# see if this already has meta-data uploaded, otherwise do nothing
156161
mc_prod_info_q = query_mcprodinfo(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag)
157-
if mc_prod_info_q == None or force_overwrite == True:
162+
# a writer that does not know the first orbit (e.g. the harvester) keeps the stored one
163+
if mc_prod_info.FirstOrbitMC == None and mc_prod_info_q != None and mc_prod_info_q.FirstOrbitMC != None:
164+
mc_prod_info = dataclasses.replace(mc_prod_info, FirstOrbitMC = mc_prod_info_q.FirstOrbitMC, Hash = None)
165+
# also publish when the first orbit differs, e.g. for a run re-done with different anchoring
166+
first_orbit_changed = mc_prod_info_q != None and mc_prod_info.FirstOrbitMC != None and mc_prod_info_q.FirstOrbitMC != mc_prod_info.FirstOrbitMC
167+
if mc_prod_info_q == None or force_overwrite == True or first_orbit_changed:
158168
# could make this depend on hash values in future
159-
upload_mcprodinfo_meta(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag, dataclasses.asdict(mc_prod_info))
169+
return upload_mcprodinfo_meta(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag, dataclasses.asdict(mc_prod_info))
160170

Lines changed: 320 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,320 @@
1+
#!/usr/bin/env python3
2+
3+
# A tool to add the key FirstOrbitMC to the MCProdInfo objects of an existing MC production.
4+
# FirstOrbitMC is the first orbit the MC used; analysis (AggregatedRunInfo::adjust_from_MC)
5+
# aligns its timeframes with it.
6+
#
7+
# The value is read per run from the log archives of the run's own GRID jobs, since a production
8+
# may contain runs made with different software. Runs without a reachable log archive can be
9+
# filled, and others cross-checked, by rerunning the anchoring step with the run's software.
10+
#
11+
# 1. scan (needs alien.py and a GRID token): read two log archives per run
12+
# mcprodinfo_firstorbit.py scan --tag LHC26j17 -o LHC26j17.json
13+
# 2. rerun (needs CVMFS): fill runs without archive and cross-check a sample
14+
# mcprodinfo_firstorbit.py rerun LHC26j17.json --crosscheck 3
15+
# 3. apply (needs a GRID token with the right role): publish new MCProdInfo objects
16+
# mcprodinfo_firstorbit.py apply LHC26j17.json --comment "FirstOrbitMC added because of O2-7335" --yes
17+
# The apply step is a dry run unless --yes is given.
18+
19+
import argparse
20+
import collections
21+
import concurrent.futures
22+
import json
23+
import os
24+
import random
25+
import re
26+
import subprocess
27+
import sys
28+
import tempfile
29+
import zipfile
30+
from urllib.parse import parse_qs, quote, urlparse
31+
32+
import requests
33+
import urllib3
34+
35+
from mcprodinfo_ccdb_upload import MCProdInfo, publish_MCProdInfo
36+
37+
MONALISA = "https://alimonitor.cern.ch"
38+
39+
40+
def token_cert():
41+
key = os.environ.get("JALIEN_TOKEN_KEY")
42+
cert = os.environ.get("JALIEN_TOKEN_CERT")
43+
if key is None and cert is None:
44+
uid = os.getuid()
45+
cert, key = f"/tmp/tokencert_{uid}.pem", f"/tmp/tokenkey_{uid}.pem"
46+
return cert, key
47+
48+
49+
def list_mcprodinfo(ccdb_url, username, tag):
50+
"""Newest MCProdInfo metadata per run of a production."""
51+
url = f"{ccdb_url}/browse/Users/{username[0]}/{username}/MCProdInfo/{tag}"
52+
r = requests.get(url, headers={"Accept": "application/json"}, timeout=120)
53+
r.raise_for_status()
54+
newest = {}
55+
for o in r.json().get("objects", []):
56+
run = int(o["RunNumber"])
57+
if run not in newest or int(o["createTime"]) > int(newest[run]["createTime"]):
58+
newest[run] = o
59+
return newest
60+
61+
62+
def newest_mcprodinfo(ccdb_url, username, tag, run):
63+
"""Newest MCProdInfo object of one run, or None."""
64+
url = f"{ccdb_url}/browse/Users/{username[0]}/{username}/MCProdInfo/{tag}/{run}/{run + 1}"
65+
r = requests.get(url, headers={"Accept": "application/json"}, cert=token_cert(), verify=False, timeout=120)
66+
r.raise_for_status()
67+
objs = r.json().get("objects", [])
68+
return max(objs, key=lambda o: int(o["createTime"])) if objs else None
69+
70+
71+
def masterjobs(tag):
72+
"""Masterjob ids and output directories of a production, per run (from MonALISA)."""
73+
cert = token_cert()
74+
r = requests.get(f"{MONALISA}/MC/prodDetails.jsp?tag={quote(tag)}", cert=cert, verify=False,
75+
allow_redirects=False, timeout=120)
76+
jobtype = parse_qs(urlparse(r.headers.get("Location", "")).query).get("filter_jobtype", [None])[0]
77+
if not jobtype:
78+
raise RuntimeError(f"no MonALISA jobtype for production {tag}")
79+
r = requests.get(f"{MONALISA}/job_events.jsp?timesel=0&owner=aliprod&filter_jobtype={quote(jobtype)}&res_path=json",
80+
cert=cert, verify=False, timeout=300)
81+
jobs = collections.defaultdict(dict)
82+
for mjid, entry in r.json().items():
83+
run, outdir = entry.get("run_no"), entry.get("outputdir")
84+
if str(run).isdigit() and outdir:
85+
jobs[int(run)][str(mjid)] = outdir.rstrip("/")
86+
return jobs
87+
88+
89+
def masterjob_package(mjid):
90+
"""Packages of a masterjob, from its JDL."""
91+
m = re.search(r"Packages\s*=\s*\{([^}]*)\}", alien("ps", "-jdl", mjid).stdout)
92+
return ",".join(re.findall(r'"([^"]+)"', m.group(1))) if m else None
93+
94+
95+
def alien(*args, timeout=300):
96+
return subprocess.run(["alien.py", *args], capture_output=True, text=True, timeout=timeout)
97+
98+
99+
def read_job(jobdir, run):
100+
"""Package, FirstOrbitMC and data first orbit from the output of one job, or None.
101+
102+
The anchoring log and stdout are read from log_archive.zip, or from the job directory
103+
when a production stores them there as plain files."""
104+
stdout, log = "", ""
105+
with tempfile.TemporaryDirectory() as d:
106+
local = os.path.join(d, "log_archive.zip")
107+
if alien("cp", f"alien://{jobdir}/log_archive.zip", f"file://{local}").returncode == 0:
108+
try:
109+
z = zipfile.ZipFile(local)
110+
stdout = z.read("stdout").decode(errors="ignore") if "stdout" in z.namelist() else ""
111+
ts = [n for n in z.namelist() if n.startswith("timestampsampling")]
112+
log = z.read(ts[0]).decode(errors="ignore") if ts else ""
113+
except (zipfile.BadZipFile, KeyError):
114+
pass
115+
for name in ("stdout", f"timestampsampling_{run}.log"):
116+
if (name == "stdout" and stdout) or (name != "stdout" and log):
117+
continue
118+
plain = os.path.join(d, name)
119+
if alien("cp", f"alien://{jobdir}/{name}", f"file://{plain}").returncode == 0:
120+
text = open(plain, errors="ignore").read()
121+
if name == "stdout":
122+
stdout = text
123+
else:
124+
log = text
125+
fo = re.search(r"Determined first orbit to be:\s+(\d+)", log)
126+
os_ = re.search(r"^OS\s+(\d+)", log, re.M)
127+
pkg = re.search(r"ALIEN_JDL_PACKAGES=(\S+)", stdout)
128+
if not (fo and os_):
129+
return None
130+
return {"lfn": jobdir, "package": pkg.group(1) if pkg else None,
131+
"FirstOrbitMC": int(fo.group(1)), "FirstOrbitData": int(os_.group(1))}
132+
133+
134+
def scan_run(run, outdirs, nsamples):
135+
"""Reads up to nsamples log archives of a run, spread over its masterjobs."""
136+
candidates = []
137+
for outdir in sorted(outdirs):
138+
splits = sorted(s.strip("/") for s in alien("ls", outdir).stdout.split() if re.fullmatch(r"\d{3}/?", s))
139+
if splits:
140+
candidates.append([f"{outdir}/{s}" for s in (splits[:2] + splits[-2:])])
141+
samples = []
142+
# round robin over the masterjobs, so that different masterjobs of one run are compared
143+
for i in range(4):
144+
for c in candidates:
145+
if len(samples) >= nsamples:
146+
break
147+
if i < len(c):
148+
s = read_job(c[i], run)
149+
if s:
150+
samples.append(s)
151+
return samples, bool(candidates)
152+
153+
154+
def scan(args):
155+
meta = list_mcprodinfo(args.ccdb, args.username, args.tag)
156+
jobs = masterjobs(args.tag)
157+
runs = sorted(r for r in meta if not args.runs or r in args.runs)
158+
rows = {}
159+
160+
def one(run):
161+
mjs = jobs.get(run, {})
162+
packages = {mj: masterjob_package(mj) for mj in mjs}
163+
return run, mjs, packages, scan_run(run, mjs.values(), args.samples)
164+
165+
with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex:
166+
for run, mjs, mjpackages, (samples, has_output) in ex.map(one, runs):
167+
optf = int(meta[run]["OrbitsPerTF"])
168+
row = {"run": run, "OrbitsPerTF": optf, "masterjobs": mjpackages, "samples": samples, "meta": meta[run]}
169+
firsts = {(s["FirstOrbitMC"], s["FirstOrbitData"]) for s in samples}
170+
packages = {p for p in mjpackages.values() if p} | {s["package"] for s in samples if s["package"]}
171+
if len(packages) == 1:
172+
row["package"] = next(iter(packages))
173+
if not mjs:
174+
row["status"] = "NO_JOBS"
175+
elif not has_output:
176+
row["status"] = "NO_OUTPUT"
177+
elif len(packages) > 1 or len(firsts) > 1:
178+
row["status"] = "MIXED"
179+
elif not samples:
180+
row["status"] = "NO_ARCHIVE"
181+
else:
182+
(row["FirstOrbitMC"], row["FirstOrbitData"]), = firsts
183+
row["shift"] = (row["FirstOrbitMC"] - row["FirstOrbitData"]) % optf
184+
row["status"] = "OK"
185+
rows[run] = row
186+
print(f"{run} OrbitsPerTF={optf} {row['status']} samples={len(samples)} "
187+
f"package={row.get('package')} FirstOrbitMC={row.get('FirstOrbitMC')} shift={row.get('shift')}", flush=True)
188+
with open(args.output, "w") as f:
189+
json.dump({"tag": args.tag, "username": args.username, "rows": [rows[r] for r in runs]}, f, indent=1)
190+
status = collections.Counter(r["status"] for r in rows.values())
191+
print(f"{args.output}: {dict(status)}")
192+
return 0
193+
194+
195+
def rerun_anchoring(package, run, orbits_per_tf, tag, timeout=900):
196+
"""Runs the anchoring step for one run; returns (data first orbit, MC first orbit)."""
197+
pkg = package.split("::")[-1]
198+
with tempfile.TemporaryDirectory(prefix=f"anchor_{run}_") as wd:
199+
inner = (f"python3 $O2DPG_ROOT/MC/bin/o2dpg_sim_workflow_anchored.py --run-number {run} --split-id 1 "
200+
f"--prod-split 100 --cycle 0 -tf 1 --orbitsPerTF {orbits_per_tf} -- -gen pythia8 -proc inel "
201+
f"-productionTag {tag} -e TGeant4 -j 8 -ns 1")
202+
cmd = ["/cvmfs/alice.cern.ch/bin/alienv", "setenv", f"O2PDPSuite/{pkg}", "-c", "bash", "-c", inner]
203+
env = {k: v for k, v in os.environ.items() if not k.startswith("ALIEN_JDL")}
204+
p = subprocess.run(cmd, cwd=wd, env=env, capture_output=True, text=True, timeout=timeout)
205+
out = p.stdout + p.stderr
206+
os_ = re.search(r"^OS\s+(\d+)", out, re.M)
207+
fo = re.search(r"Determined first orbit to be:\s+(\d+)", out)
208+
if not (os_ and fo):
209+
raise RuntimeError(f"run {run}: anchoring output not understood (rc {p.returncode}): {out[-300:]}")
210+
return int(os_.group(1)), int(fo.group(1))
211+
212+
213+
def rerun(args):
214+
with open(args.table) as f:
215+
table = json.load(f)
216+
rows = table["rows"]
217+
todo = [r for r in rows if r["status"] == "NO_ARCHIVE"]
218+
ok = [r for r in rows if r["status"] == "OK" and r.get("package")]
219+
todo += random.Random(1).sample(ok, min(args.crosscheck, len(ok)))
220+
221+
def one(row):
222+
package = args.package or row.get("package")
223+
if not package:
224+
return row, None, "no package known"
225+
try:
226+
return row, rerun_anchoring(package, row["run"], row["OrbitsPerTF"], table["tag"]), package
227+
except Exception as e:
228+
return row, None, str(e)
229+
230+
with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex:
231+
for row, res, info in ex.map(one, todo):
232+
if res is None:
233+
print(f"{row['run']} rerun failed: {info}", flush=True)
234+
continue
235+
data_first, mc_first = res
236+
if row["status"] == "OK":
237+
agree = (mc_first, data_first) == (row["FirstOrbitMC"], row["FirstOrbitData"])
238+
row["crosscheck"] = "agree" if agree else f"DIFFER rerun {mc_first}/{data_first}"
239+
print(f"{row['run']} cross-check {row['crosscheck']}", flush=True)
240+
else:
241+
row.update(status="RERUN", package=info, FirstOrbitMC=mc_first, FirstOrbitData=data_first,
242+
shift=(mc_first - data_first) % row["OrbitsPerTF"])
243+
print(f"{row['run']} RERUN package={info} FirstOrbitMC={mc_first} shift={row['shift']}", flush=True)
244+
with open(args.table, "w") as f:
245+
json.dump(table, f, indent=1)
246+
return 0
247+
248+
249+
def apply(args):
250+
with open(args.table) as f:
251+
table = json.load(f)
252+
tag = table["tag"]
253+
username = args.username or table["username"]
254+
for row in table["rows"]:
255+
if row["status"] not in ("OK", "RERUN") or str(row.get("crosscheck", "agree")) != "agree":
256+
print(f"SKIP {tag} run {row['run']} status={row['status']} crosscheck={row.get('crosscheck')}")
257+
continue
258+
m = row["meta"]
259+
# the newest object must still be the one scanned; skip runs that already carry the value
260+
current = newest_mcprodinfo(args.ccdb, username, tag, row["run"])
261+
if current is not None and current.get("id") != m.get("id"):
262+
if str(current.get("FirstOrbitMC")) == str(row["FirstOrbitMC"]):
263+
print(f"DONE {tag} run {row['run']} already has FirstOrbitMC={row['FirstOrbitMC']}")
264+
continue
265+
print(f"ERROR: {tag} run {row['run']} has a newer object ({current.get('id')}) than the scanned one "
266+
f"({m.get('id')}), with FirstOrbitMC={current.get('FirstOrbitMC')}; scan again")
267+
return 1
268+
info = MCProdInfo(LPMProductionTag=tag,
269+
Col=m.get("Col"),
270+
IntRate=float(m["IntRate"]) if m.get("IntRate") not in (None, "None") else None,
271+
RunNumber=row["run"],
272+
OrbitsPerTF=row["OrbitsPerTF"],
273+
Comment=args.comment,
274+
McTag=m.get("McTag") if m.get("McTag") not in (None, "None") else row.get("package"),
275+
RecoTag=m.get("RecoTag") if m.get("RecoTag") != "None" else None,
276+
FirstOrbitMC=row["FirstOrbitMC"])
277+
print(f"{'PUBLISH' if args.yes else 'DRY'} {tag} run {row['run']} FirstOrbitMC={row['FirstOrbitMC']} "
278+
f"shift={row['shift']} McTag={info.McTag} -> {args.ccdb} Users/{username[0]}/{username}")
279+
if args.yes:
280+
r = publish_MCProdInfo(info, ccdb_url=args.ccdb, username=username, force_overwrite=True)
281+
where = r.headers.get("Location") or r.headers.get("Content-Location") if r is not None else None
282+
with open(args.log, "a") as log:
283+
log.write(f"{tag} {row['run']} {r.status_code if r is not None else None} {where}\n")
284+
if r is None or r.status_code not in (200, 201):
285+
print(f"ERROR: upload of run {row['run']} failed: {r.status_code if r is not None else 'no response'} "
286+
f"{(r.text[:300] if r is not None else '')}")
287+
return 1
288+
return 0
289+
290+
291+
def main():
292+
urllib3.disable_warnings()
293+
parser = argparse.ArgumentParser(description="Add FirstOrbitMC to the MCProdInfo objects of an MC production")
294+
sub = parser.add_subparsers(dest="cmd", required=True)
295+
s = sub.add_parser("scan", help="read FirstOrbitMC per run from the log archives of its jobs")
296+
s.add_argument("--tag", required=True, help="LPM production tag")
297+
s.add_argument("--runs", type=int, nargs="*", help="restrict to these runs")
298+
s.add_argument("--samples", type=int, default=2, help="log archives to read per run")
299+
s.add_argument("--ccdb", default="http://alice-ccdb.cern.ch", help="CCDB to read MCProdInfo from")
300+
s.add_argument("--username", default="aliprod", help="owner of the MCProdInfo objects")
301+
s.add_argument("-j", "--jobs", type=int, default=8, help="runs processed in parallel")
302+
s.add_argument("-o", "--output", required=True, help="output table (JSON)")
303+
r = sub.add_parser("rerun", help="rerun the anchoring for runs without archive and for a cross-check sample")
304+
r.add_argument("table", help="table written by scan, updated in place")
305+
r.add_argument("--package", help="override the O2PDPSuite package of each run (default: from its masterjob JDL)")
306+
r.add_argument("--crosscheck", type=int, default=3, help="number of runs with archive to cross-check")
307+
r.add_argument("-j", "--jobs", type=int, default=8, help="parallel anchoring reruns")
308+
a = sub.add_parser("apply", help="publish MCProdInfo objects with FirstOrbitMC from a table")
309+
a.add_argument("table", help="table written by scan")
310+
a.add_argument("--comment", required=True, help="Comment stored in the new objects")
311+
a.add_argument("--ccdb", default="https://alice-ccdb.cern.ch", help="CCDB to publish to")
312+
a.add_argument("--username", help="owner path to publish under (default: as scanned)")
313+
a.add_argument("--yes", action="store_true", help="really publish (default: dry run)")
314+
a.add_argument("--log", default="mcprodinfo_firstorbit_published.log", help="appends tag, run, HTTP status and object location per upload")
315+
args = parser.parse_args()
316+
return {"scan": scan, "rerun": rerun, "apply": apply}[args.cmd](args)
317+
318+
319+
if __name__ == "__main__":
320+
sys.exit(main())

0 commit comments

Comments
 (0)