Skip to content

Commit b7ce77e

Browse files
sawenzelclaude
andcommitted
Record FirstOrbitMC and the software version in MCProdInfo
This stores the first orbit used by an anchored MC job and its software version in its MCProdInfo object, and provides a new tool mcprodinfo_firstorbit.py that adds the first orbit to existing productions. - Analysis uses FirstOrbitMC to align its timeframes with the MC. - McTag is now filled at job time from ALIEN_JDL_PACKAGES. - A stored FirstOrbitMC is never replaced; a job with a different value keeps it and prints a warning. Only force_overwrite, as used by the tool, replaces it. - Optional fields a writer leaves empty keep their stored values, e.g. FirstOrbitMC for the harvester with --overwrite. - If the existing object cannot be read, nothing is published. - Optional metadata with value None is no longer uploaded as the string "None", and metadata values are URL-escaped. - Reading an object that lacks a required field no longer raises a TypeError. - The tool reads FirstOrbitMC per run from the log archives of the run's own jobs, found via MonALISA. - The software of each run is taken from its masterjob JDL and log archives; runs whose jobs disagree are flagged and skipped. - Runs without reachable archive are filled, and a sample cross-checked, by rerunning the anchoring with the run's software. - apply skips runs that already carry the value and stops if a run changed since the scan. Usage: mcprodinfo_firstorbit.py scan --tag LHC26h3 -o LHC26h3.json mcprodinfo_firstorbit.py rerun LHC26h3.json --crosscheck 3 mcprodinfo_firstorbit.py apply LHC26h3.json --comment "FirstOrbitMC added because of O2-7335" --yes For LHC26h3 the scan gives 60 runs made with MC-prod-2026-v12 (shift 2), 3 runs made with v14 (aligned) and 4 runs without jobs. https://its.cern.ch/jira/browse/O2-7335 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent ef75c8f commit b7ce77e

3 files changed

Lines changed: 354 additions & 5 deletions

File tree

‎MC/bin/o2dpg_sim_workflow_anchored.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -811,6 +811,7 @@ def main():
811811
IntRate =rate,
812812
RunNumber = args.run_number,
813813
OrbitsPerTF = GLOparams["OrbitsPerTF"],
814+
McTag = os.getenv("ALIEN_JDL_PACKAGES"),
814815
FirstOrbitMC = effective_first_orbit)
815816
publish_MCProdInfo(info, username = grid_user_name, ccdb_url = mcprod_ccdb_server)
816817
else:

‎MC/prodinfo/mcprodinfo_ccdb_upload.py‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import json
22
import os
33
import requests
4+
import urllib.parse
45
import subprocess
56

67
import dataclasses # to define the MCProdInfo data layout and convert it to dict
@@ -22,6 +23,7 @@ class MCProdInfo:
2223
Comment: Optional[str] = None
2324
McTag: Optional[str] = None # main software tag used
2425
RecoTag: Optional[str] = None # RecoTag (if any)
26+
FirstOrbitMC: Optional[int] = None # first orbit used by the MC; its TFs start at FirstOrbitMC + k * OrbitsPerTF
2527
Hash: Optional[str] = field(default=None)
2628

2729
def __post_init__(self):
@@ -87,14 +89,19 @@ def query_mcprodinfo(base_url, user, run_number, lpm_prod_tag, cert_dir="/tmp"):
8789
url = f"{base_url}/browse/{user_path}/MCProdInfo/{lpm_prod_tag}/{start}/{stop}"
8890

8991
response = requests.get(url, cert=(cert_path, key_path), verify=False)
90-
if response.status_code != 404:
92+
if response.status_code not in (200, 404):
93+
raise RuntimeError(f"CCDB query {url} failed with HTTP {response.status_code}")
94+
if response.status_code == 200:
9195
meta = extract_metadata_blocks_from_CCDB(response.content.decode('utf-8'))
9296
if (len(meta) > 0):
9397
def filter_known_fields(cls, data: dict) -> dict:
9498
valid_keys = {f.name for f in fields(cls)}
9599
return {k: v for k, v in data.items() if k in valid_keys}
96100

97101
clean_meta = filter_known_fields(MCProdInfo, meta[0])
102+
# an object may lack fields that are required here (not every writer sets all of them)
103+
for f in fields(MCProdInfo):
104+
clean_meta.setdefault(f.name, None)
98105
return MCProdInfo(**clean_meta)
99106

100107
return None
@@ -127,7 +134,8 @@ def upload_mcprodinfo_meta(base_url, user, run_number, lpm_prod_tag, keys, cert_
127134
key_path = os.path.join(cert_dir, f"tokenkey_{uid}.pem")
128135

129136
# Build full URL
130-
query = "/".join(f"{k}={v}" for k, v in keys.items())
137+
optional = {f.name for f in fields(MCProdInfo) if f.default is not dataclasses.MISSING}
138+
query = "/".join(f"{k}={urllib.parse.quote(str(v), safe='@:,')}" for k, v in keys.items() if v is not None or k not in optional)
131139
user_path = 'Users/' + user[0] + '/' + user
132140
start = run_number
133141
stop = run_number + 1
@@ -153,8 +161,28 @@ def publish_MCProdInfo(mc_prod_info, ccdb_url = "https://alice-ccdb.cern.ch", us
153161
return
154162

155163
# see if this already has meta-data uploaded, otherwise do nothing
156-
mc_prod_info_q = query_mcprodinfo(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag)
157-
if mc_prod_info_q == None or force_overwrite == True:
164+
try:
165+
mc_prod_info_q = query_mcprodinfo(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag)
166+
except (RuntimeError, requests.RequestException) as e:
167+
print(f"Not publishing MCProdInfo, the existing one could not be read: {e}")
168+
return None
169+
# optional fields the writer leaves empty (e.g. FirstOrbitMC for the harvester) keep their stored values
170+
if mc_prod_info_q != None:
171+
kept = {f.name: getattr(mc_prod_info_q, f.name) for f in fields(MCProdInfo)
172+
if f.name != "Hash" and f.default is not dataclasses.MISSING
173+
and getattr(mc_prod_info, f.name) == None and getattr(mc_prod_info_q, f.name) != None}
174+
if kept:
175+
mc_prod_info = dataclasses.replace(mc_prod_info, Hash = None, **kept)
176+
# a stored first orbit is never replaced, since it describes the data already produced for this run;
177+
# only force_overwrite (e.g. mcprodinfo_firstorbit.py apply) replaces it
178+
stored_first_orbit = mc_prod_info_q.FirstOrbitMC if mc_prod_info_q != None else None
179+
if stored_first_orbit != None and mc_prod_info.FirstOrbitMC != None and stored_first_orbit != mc_prod_info.FirstOrbitMC:
180+
action = "overwriting it (forced)" if force_overwrite else "keeping the stored value"
181+
print(f"WARNING: MCProdInfo of {mc_prod_info.LPMProductionTag} run {mc_prod_info.RunNumber} has FirstOrbitMC {stored_first_orbit}, "
182+
f"this job uses {mc_prod_info.FirstOrbitMC}; its timeframes will be misaligned in analysis. {action}")
183+
# publish when there is no object yet, or when the stored one lacks the first orbit
184+
adds_first_orbit = mc_prod_info_q != None and stored_first_orbit == None and mc_prod_info.FirstOrbitMC != None
185+
if mc_prod_info_q == None or force_overwrite == True or adds_first_orbit:
158186
# could make this depend on hash values in future
159-
upload_mcprodinfo_meta(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag, dataclasses.asdict(mc_prod_info))
187+
return upload_mcprodinfo_meta(ccdb_url, username, mc_prod_info.RunNumber, mc_prod_info.LPMProductionTag, dataclasses.asdict(mc_prod_info))
160188

0 commit comments

Comments
 (0)