From 25624dda2bdcb9a6a904dc3b5a9713095e015a2b Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Tue, 10 Nov 2026 12:57:20 +0100 Subject: [PATCH 01/10] getCCDBTimeMachineTimestamp.py: do second CCDB query before uploading new object --- GRID/utils/getCCDBTimeMachineTimestamp.py | 37 +++++++++++++++-------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index f5b4619e6..ae50d1463 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -211,7 +211,7 @@ def query_CCDBTimeMachineTS(base_url, user, period, passname): Queries CCDBTimeMachineTS from CCDB for a given period and pass. Returns: - obtained NotAfter timestamp (int) or None + (tuple) (obtained NotAfter timestamp (int), True) or (None, False) """ # Construct user ID-specific cert and key paths cert_path, key_path = make_cert_paths() @@ -235,9 +235,9 @@ def filter_known_fields(cls, data: dict) -> dict: print(f"[getCCDBTimeMachineTimestamp] Found object for {clean_meta['Period']}, Pass: {clean_meta['Pass']}, " f"NotAfter: {clean_meta['NotAfter']} ({human(clean_meta['NotAfter'])}), uploaded by LPMProductionTag {clean_meta['LPMProductionTag']}", file=sys.stderr) - return clean_meta['NotAfter'] + return clean_meta['NotAfter'], True - return None + return None, False def upload_CCDBTimeMachineTS(base_url, user, keys): @@ -288,20 +288,31 @@ def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, Returns: int: Either the queried or the estimated NotAfter timestamp """ + found = False if not skip_ccdb_query: - not_after = query_CCDBTimeMachineTS(ccdb_url, username, period, passname) - else: - not_after = None + not_after, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname) - if not_after is not None and force_overwrite == True: - print("[getCCDBTimeMachineTimestamp] Existing object will be overwritten", file=sys.stderr) - - if not_after == None or force_overwrite == True: + if not found or force_overwrite: print("[getCCDBTimeMachineTimestamp] Estimating CCDBTimeMachineTS for " f"LPMProductionTag: {production_tag}, Period: {period}, Pass: {passname}", file=sys.stderr) not_after = get_NotAfter(year, period, run, passname, samples, ccdb_check_path, ccdb_check) - time_machine_ts = CCDBTimeMachineTS(production_tag, period, passname, not_after) + if publish_to_ccdb: + if skip_ccdb_query: + print("[getCCDBTimeMachineTimestamp] CCDB query was skipped, a newer object may be " + "uploaded on top of an existing one", file=sys.stderr) + elif not found: + # check again in case another subjob uploaded an object in the meantime + not_after_q, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname) + + if found: + if force_overwrite: + print("[getCCDBTimeMachineTimestamp] Forcing upload of a newer object " + "on top of the existing one", file=sys.stderr) + else: + return not_after_q + + time_machine_ts = CCDBTimeMachineTS(production_tag, period, passname, not_after) upload_CCDBTimeMachineTS(ccdb_url, username, dataclasses.asdict(time_machine_ts)) return not_after @@ -335,11 +346,11 @@ def main(): help="overwrite the current CCDBTimeMachineTS object in the CCDB if one exists") parser.add_argument("--ccdb-url", default="https://alice-ccdb.cern.ch", help="ccdb host for the upload") - parser.add_argument("--username", default="aliprod", + parser.add_argument("--username", default=os.environ.get("JALIEN_USER"), help="username to be used for the upload to CCDB") args = parser.parse_args() - missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)] + missing = [n for n in ("production_tag", "year", "period", "run", "passname", "username") if not getattr(args, n)] if missing: parser.error("missing required value(s): " + ", ".join(missing)) From 5b70a3ecda29ab421b139fded2fc19d69432a32a Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Tue, 10 Nov 2026 12:57:20 +0100 Subject: [PATCH 02/10] Get CCDB time machine timestamp from CCDB by default in anchorMC.sh Related JIRA ticket: https://its.cern.ch/jira/browse/O2-6543 This will use the script GRID/utils/getCCDBTimeMachineTimestamp.py in anchorMC.sh to obtain the CCDB time machine timestamp from the CCDB (or estimate it from the given anchored apass output files). This is now the default behavior. If the timestamp cannot be obtained, anchorMC.sh exits with an error. If set, the JDL vairable `ALIEN_JDL_CCDB_CONDITION_NOT_AFTER` can still be used to set the `--condition-not-after` timestamp by hand instead of obtaining it from CCDB. The default object path for the time machine timestamp is `Users/a/aliprod/CCDBTimeMachineTS`. All objects are uploaded with validity `1:2`, and the metadata contains the pass, period, NotAfter and LPMProductionTag (to track the productions which uploaded the object). So there should be one object per reconstruction pass per period, valid for all runs of the given pass and period. Some test examples are already uploaded in http://ccdb-test.cern.ch:8080/browse/Users/e/ehellbar/CCDBTimeMachineTS. This PR contains two commits: 1) in getCCDBTimeMachineTimestamp.py - add a second CCDB query to check for existing objects right before the upload of a new object, to check if between the last empty query and now (after the estimation of the not-after timestamp from the reco output) another job was faster and already uploaded the object. This is an attempt to avoid duplicate objects for the same period/pass uploaded by a single LPMProductionTag. NB: a more elegent way to create these CCDBTimeMachineTS objects by default could be at the beginning or the end of the actual data reconstruction pass. 2) use getCCDBTimeMachineTimestamp.py in anchorMC.sh to obtain the CCDB time machine timestamp by default, keeping the current `ALIEN_JDL_CCDB_CONDITION_NOT_AFTER` option to enforce it by hand I mark this as WIP since the following is still missing: - treatment of MC specific CCDB objects: There is a handful of MC CCDB paths (TPC corrections, GLO/Param/MatLUT) for which the objects are uploaded independently of the data calibration objects, so the NotAfter as it is obtained here might invalidate the correct object to use. With Giulio, Ruben and Costin, we are working on an elegant way to handle this. - I still need to run a small test on the grid to finally validate the implementation. --- MC/run/ANCHOR/anchorMC.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 00615a253..679171e28 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -354,7 +354,29 @@ remainingargs="${ALIEN_JDL_ANCHOR_SIM_OPTIONS} ${remainingargs} --anchor-config ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE=${ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE:-RECO} remainingargs="${remainingargs} ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG:+--alternative-reco-software ${PWD}/env_async.env@${ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE}}" # potentially add CCDB timemachine timestamp -remainingargs="${remainingargs} ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}}" +if [[ -n ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER} ]]; then + # override CCDB time machine timestamp by hand, use e.g. 3385078236000 to effectively disable it + ARG_CONDITION_NOT_AFTER="--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}" +else + # options to set CCDB url, disable upload and force overwrite of an existing timestamp; default to production CCDB, uploading the object and not overwriting + : ${CCDBTIMEMACHINE_CCDB_URL:=https://alice-ccdb.cern.ch} + [[ ${CCDBTIMEMACHINE_SKIP_UPLOAD:-} != "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --publish-to-ccdb" + [[ ${CCDBTIMEMACHINE_FORCE_OVERWRITE:-} == "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --force-overwrite" + + # obtain CCDB time machine timestamp from CCDB or, if not available, estimate it from the creation time of the reconstruction output files + CCDBTIMEMACHINE_CMD="${O2DPG_ROOT}/GRID/utils/getCCDBTimeMachineTimestamp.py --ccdb-url $CCDBTIMEMACHINE_CCDB_URL ${CCDBTIMEMACHINE_OPTIONS:-}" + echo "Running command to obtain CCDB time machine timestamp: ${CCDBTIMEMACHINE_CMD}" + CCDBTIMEMACHINE_TS=$(${CCDBTIMEMACHINE_CMD}) + + # return an error if the timestamp could not be obtained + if [[ -n ${CCDBTIMEMACHINE_TS} ]]; then + ARG_CONDITION_NOT_AFTER="--condition-not-after ${CCDBTIMEMACHINE_TS}" + else + echo_error "Could not obtain CCDB time machine timestamp. Exiting." + exit 1 + fi +fi +remainingargs="${remainingargs} ${ARG_CONDITION_NOT_AFTER:-}" echo_info "baseargs passed to o2dpg_sim_workflow_anchored.py: ${baseargs}" echo_info "remainingargs forwarded to o2dpg_sim_workflow.py: ${remainingargs}" From 0a247225830222fa11c290f52ca6a441729b2028 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Tue, 10 Nov 2026 13:34:37 +0100 Subject: [PATCH 03/10] use argument instead of env var to set condition-not-after for global alignment geometry --- MC/bin/o2dpg_sim_workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 24bf25aba..019ccc402 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -596,7 +596,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True): fi # fetch the global alignment geometry -${{O2_ROOT}}/bin/o2-create-aligned-geometry-workflow ${{ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after $ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}} \ +${{O2_ROOT}}/bin/o2-create-aligned-geometry-workflow --condition-not-after {args.condition_not_after} \ --configKeyValues "HBFUtils.startTime={args.timestamp}" -b --run ${{REMAP_OPT}} # copy the object into the CCDB cache @@ -1378,13 +1378,13 @@ def getDigiTaskName(det): if not isActive('CTP'): # CTP digits won't be produced for this timeframe (CTP not in the readout detector list) tpc_corr_scaling_options += ' --disable-ctp-lumi-request' - + # why not simply? # tpc_corr_scaling_options = ('--lumi-type 1', '')[tpcDistortionType != 0] # at the moment MC is not ready for the sector edge fluctuations (RS: and does not use IDC but --disable-IDC-scalers can be added only once it is added by the CorrectionMapsOptions::addGlobalOptions) tpc_corr_scaling_options += option_if_available('o2-tpc-reco-workflow', '--disable-sec-edge-fluc-correction', envfile=async_envfile) - + #<--------- TPC reco task if includeTPCSyncMode: tpcSyncreconeeds = tpcreconeeds.copy() From dee2b19e351ce04d2f012405f301451f36a8739d Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 04/10] getCCDBTimeMachineTimestamp.py: refactor printing of log messages --- GRID/utils/getCCDBTimeMachineTimestamp.py | 97 +++++++++++++---------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index ae50d1463..a197a651b 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -91,6 +91,22 @@ def human(ts_ms): return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") +def print_log(severity, message, file): + print(f"[getCCDBTimeMachineTimestamp] [{severity}] {message}", file=file) + + +def print_info(message, file=sys.stderr): + print_log("Info", message, file) + + +def print_warning(message, file=sys.stderr): + print_log("Warning", message, file) + + +def print_error(message, file=sys.stderr): + print_log("Error", message, file) + + def get_ccdb_objects(url, timeout = 15): req = urllib.request.Request(url, headers={"Accept": "application/json"}) try: @@ -120,7 +136,7 @@ def ccdb_cross_check(run, path, not_after, timeout=15): objects = get_ccdb_objects(url) if not objects: - return {**result, "error": "no CCDB object covers this instant"} + result.update({"error": "no CCDB object covers this instant"}) as_of = [o for o in objects if o.get("Created", 0) <= not_after] newest = max(objects, key=lambda o: o.get("Created", 0)) @@ -128,7 +144,21 @@ def ccdb_cross_check(run, path, not_after, timeout=15): if as_of: result["as_of"] = max(as_of, key=lambda o: o.get("Created", 0)) result["outdated"] = newest["Created"] > result["as_of"]["Created"] - return result + + check_timestamp = f"/{result['timestamp']}" if result else "" + if result is None or "error" in result: + reason = result["error"] if result else "unknown error" + print_warning(f"CCDB cross-check on {path}{check_timestamp} skipped: {reason}") + elif result["as_of"] is None: + print_info(f"CCDB cross-check on {path}{check_timestamp}: no object existed " + f"yet at the recommended timestamp (newest is {describe_object(result['newest'])})") + else: + print_info(f"CCDB cross-check on {path}{check_timestamp}: as-of object is " + f"{describe_object(result['as_of'])}") + if result["outdated"]: + print_info(f"CCDB's current object for this window is newer " + f"({describe_object(result['newest'])}) - it postdates this pass's processing and " + f"was not what this run actually saw.") def describe_object(o): @@ -157,13 +187,14 @@ def get_NotAfter(year, period, run, passname, samples=9, ccdb_check_path="TPC/Ca int: CCDBTimeMachineTS NotAfter timestamp """ pass_dir = f"/alice/data/{year}/{period}/{run}/{passname}/" - print(f"[getCCDBTimeMachineTimestamp] Looking for TF output under {pass_dir}", file=sys.stderr) + print_info(f"Looking for TF output under {pass_dir}") matches = find_tf_output_files(pass_dir, run) if not matches: - sys.exit(f"[getCCDBTimeMachineTimestamp] No o2_ctf_run{int(run):08d}_orbit* output found " - f"under {pass_dir}. Check year/period/run/pass, or this run+pass may not use this " - f"catalog layout. Refusing to guess a timestamp.") + print_error(f"No o2_ctf_run{int(run):08d}_orbit* output found under {pass_dir}. " + f"Check year/period/run/pass, or this run+pass may not use this " + f"catalog layout. Refusing to guess a timestamp. Exiting...") + sys.exit(1) sampled = sample_evenly(matches, samples) ctimes = [] @@ -171,37 +202,21 @@ def get_NotAfter(year, period, run, passname, samples=9, ccdb_check_path="TPC/Ca ctime = stat_ctime(lfn) if ctime is not None: ctimes.append(ctime) - print(f"[getCCDBTimeMachineTimestamp] {human(ctime)} {lfn}", file=sys.stderr) + print_info(f"{human(ctime)} {lfn}") if len(ctimes) < 2: - sys.exit(f"[getCCDBTimeMachineTimestamp] Only got {len(ctimes)} usable timestamp(s) out of " - f"{len(sampled)} sampled files ({len(matches)} found in total); refusing to guess.") + print_error(f"Only got {len(ctimes)} usable timestamp(s) out of {len(sampled)} " + f"sampled files ({len(matches)} found in total); refusing to guess. Exiting...") + sys.exit(1) not_after = int(statistics.median(ctimes)) spread_h = (max(ctimes) - min(ctimes)) / 3.6e6 - print(f"[getCCDBTimeMachineTimestamp] {len(ctimes)} samples out of {len(matches)} TF outputs found; " - f"span {human(min(ctimes))} .. {human(max(ctimes))} ({spread_h:.1f} h)", file=sys.stderr) - print(f"[getCCDBTimeMachineTimestamp] Recommended NotAfter: {not_after} ({human(not_after)})", - file=sys.stderr) + print_info(f"{len(ctimes)} samples out of {len(matches)} TF outputs found; " + f"span {human(min(ctimes))} .. {human(max(ctimes))} ({spread_h:.1f} h)") + print_info(f"Recommended NotAfter: {not_after} ({human(not_after)})") if ccdb_check: - check = ccdb_cross_check(run, ccdb_check_path, not_after) - check_timestamp = f"/{check['timestamp']}" if check else "" - if check is None or "error" in check: - reason = check["error"] if check else "unknown error" - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}{check_timestamp} skipped: {reason}", - file=sys.stderr) - elif check["as_of"] is None: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}{check_timestamp}: no object " - f"existed yet at the recommended timestamp (earliest is {describe_object(check['newest'])})", - file=sys.stderr) - else: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}{check_timestamp}: as-of object is " - f"{describe_object(check['as_of'])}", file=sys.stderr) - if check["outdated"]: - print(f"[getCCDBTimeMachineTimestamp] note: CCDB's current object for this window is newer " - f"({describe_object(check['newest'])}) - it postdates this pass's processing and was " - f"not what this run actually saw.", file=sys.stderr) + ccdb_cross_check(run, ccdb_check_path, not_after) return not_after @@ -221,7 +236,7 @@ def query_CCDBTimeMachineTS(base_url, user, period, passname): user_path = 'Users/' + user[0] + '/' + user url = f"{base_url}/browse/{user_path}/CCDBTimeMachineTS/{query}" - print(f"[getCCDBTimeMachineTimestamp] Checking if CCDBTimeMachineTS already exists: {url}", file=sys.stderr) + print_info(f"Checking if CCDBTimeMachineTS already exists: {url}") response = requests.get(url, cert=(cert_path, key_path), verify=False) if response.status_code == 200: @@ -232,9 +247,9 @@ def filter_known_fields(cls, data: dict) -> dict: return {k: v for k, v in data.items() if k in valid_keys} clean_meta = filter_known_fields(CCDBTimeMachineTS, meta[0]) - print(f"[getCCDBTimeMachineTimestamp] Found object for {clean_meta['Period']}, Pass: {clean_meta['Pass']}, " - f"NotAfter: {clean_meta['NotAfter']} ({human(clean_meta['NotAfter'])}), uploaded by LPMProductionTag {clean_meta['LPMProductionTag']}", - file=sys.stderr) + print_info(f"Found object for {clean_meta['Period']}, Pass: {clean_meta['Pass']}, " + f"NotAfter: {clean_meta['NotAfter']} ({human(clean_meta['NotAfter'])}), " + f"uploaded by LPMProductionTag {clean_meta['LPMProductionTag']}") return clean_meta['NotAfter'], True return None, False @@ -262,7 +277,7 @@ def upload_CCDBTimeMachineTS(base_url, user, keys): stop = start + 1 url = f"{base_url}/{user_path}/CCDBTimeMachineTS/{start}/{stop}/{query}" - print(f"[getCCDBTimeMachineTimestamp] Uploading CCDBTimeMachineTS: {url}", file=sys.stderr) + print_info(f"Uploading CCDBTimeMachineTS: {url}") # Prepare request with open(empty_file, 'rb') as f: @@ -272,7 +287,7 @@ def upload_CCDBTimeMachineTS(base_url, user, keys): os.remove(empty_file) if response.status_code != 201: - print(f"[getCCDBTimeMachineTimestamp] Upload failed, response: {response.status_code}", file=sys.stderr) + print_warning(f"Upload failed, response: {response.status_code}") def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, @@ -293,22 +308,20 @@ def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, not_after, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname) if not found or force_overwrite: - print("[getCCDBTimeMachineTimestamp] Estimating CCDBTimeMachineTS for " - f"LPMProductionTag: {production_tag}, Period: {period}, Pass: {passname}", file=sys.stderr) + print_info("Estimating CCDBTimeMachineTS for " + f"LPMProductionTag: {production_tag}, period: {period}, rass: {passname}, run: {run}") not_after = get_NotAfter(year, period, run, passname, samples, ccdb_check_path, ccdb_check) if publish_to_ccdb: if skip_ccdb_query: - print("[getCCDBTimeMachineTimestamp] CCDB query was skipped, a newer object may be " - "uploaded on top of an existing one", file=sys.stderr) + print_info("CCDB query was skipped, a newer object may be uploaded on top of an existing one") elif not found: # check again in case another subjob uploaded an object in the meantime not_after_q, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname) if found: if force_overwrite: - print("[getCCDBTimeMachineTimestamp] Forcing upload of a newer object " - "on top of the existing one", file=sys.stderr) + print_info("Forcing upload of a newer object on top of the existing one") else: return not_after_q From 84e05062ef5a7f48e6bd837bfc8fea792804e66f Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 05/10] getCCDBTimeMachineTimestamp.py: do CCDB cross check for the final not_after timestamp --- GRID/utils/getCCDBTimeMachineTimestamp.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index a197a651b..75743cfc9 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -177,11 +177,9 @@ def make_cert_paths(cert_dir="/tmp"): return cert_path, key_path -def get_NotAfter(year, period, run, passname, samples=9, ccdb_check_path="TPC/Calib/CorrectionMapV2", ccdb_check=True): +def get_NotAfter(year, period, run, passname, samples=9): """ Estimates the CCDBTimeMachine NotAfter timestamp from the output of the production of a given period/run/pass. - Cross-checks the estimated NotAfter timestamp vs the creation time of a CCDB object which is currently - valid for the given run. Returns: int: CCDBTimeMachineTS NotAfter timestamp @@ -215,9 +213,6 @@ def get_NotAfter(year, period, run, passname, samples=9, ccdb_check_path="TPC/Ca f"span {human(min(ctimes))} .. {human(max(ctimes))} ({spread_h:.1f} h)") print_info(f"Recommended NotAfter: {not_after} ({human(not_after)})") - if ccdb_check: - ccdb_cross_check(run, ccdb_check_path, not_after) - return not_after @@ -291,8 +286,7 @@ def upload_CCDBTimeMachineTS(base_url, user, keys): def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, - samples=9, ccdb_check_path="TPC/Calib/CorrectionMapV2", ccdb_check=True, - ccdb_url="https://alice-ccdb.cern.ch", username="aliprod", + samples=9, ccdb_url="https://alice-ccdb.cern.ch", username="aliprod", force_overwrite=False, publish_to_ccdb=True, skip_ccdb_query=False): """ Makes a query to CCDB for a CCDBTimeMachineTS for given period and pass. @@ -310,7 +304,7 @@ def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, if not found or force_overwrite: print_info("Estimating CCDBTimeMachineTS for " f"LPMProductionTag: {production_tag}, period: {period}, rass: {passname}, run: {run}") - not_after = get_NotAfter(year, period, run, passname, samples, ccdb_check_path, ccdb_check) + not_after = get_NotAfter(year, period, run, passname, samples) if publish_to_ccdb: if skip_ccdb_query: @@ -368,8 +362,11 @@ def main(): parser.error("missing required value(s): " + ", ".join(missing)) not_after = get_CCDBTimeMachineTS(args.production_tag, args.year, args.period, args.run, args.passname, - args.samples, args.ccdb_check_path, not args.no_ccdb_check, args.ccdb_url, args.username, - args.force_overwrite, args.publish_to_ccdb, args.skip_ccdb_query) + args.samples, args.ccdb_url, args.username, + args.force_overwrite, args.publish_to_ccdb, args.skip_ccdb_query) + if not args.no_ccdb_check: + ccdb_cross_check(args.run, args.ccdb_check_path, not_after) + print(not_after) From 8e09c7b022b04448148c819b94ef490e88a13dd1 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 06/10] getCCDBTimeMachineTimestamp.py: simplify ccdb_cross_check --- GRID/utils/getCCDBTimeMachineTimestamp.py | 40 +++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index 75743cfc9..3dacd7cdd 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -126,43 +126,41 @@ def get_run_start_tf(run): return objects[0]["STF"] -def ccdb_cross_check(run, path, not_after, timeout=15): +def ccdb_cross_check(run, path, not_after): """Report which object a correctly time-machine-pinned query at not_after would actually resolve to, and flag if CCDB has since been updated for the same validity window. Never raises; returns None on any failure.""" timestamp = get_run_start_tf(run) url = f"{CCDB_HOST}/browse/{path}/{timestamp}" - result = {"timestamp": timestamp} objects = get_ccdb_objects(url) if not objects: - result.update({"error": "no CCDB object covers this instant"}) + print_warning(f"CCDB cross-check on {path}/{timestamp} skipped: " + "no CCDB object covers this instant") + return - as_of = [o for o in objects if o.get("Created", 0) <= not_after] + as_of_candidates = [o for o in objects if o.get("Created", 0) <= not_after] newest = max(objects, key=lambda o: o.get("Created", 0)) - result.update({"newest": newest, "as_of": None, "outdated": False}) - if as_of: - result["as_of"] = max(as_of, key=lambda o: o.get("Created", 0)) - result["outdated"] = newest["Created"] > result["as_of"]["Created"] - - check_timestamp = f"/{result['timestamp']}" if result else "" - if result is None or "error" in result: - reason = result["error"] if result else "unknown error" - print_warning(f"CCDB cross-check on {path}{check_timestamp} skipped: {reason}") - elif result["as_of"] is None: - print_info(f"CCDB cross-check on {path}{check_timestamp}: no object existed " - f"yet at the recommended timestamp (newest is {describe_object(result['newest'])})") + as_of = None + outdated = False + if as_of_candidates: + as_of = max(as_of_candidates, key=lambda o: o.get("Created", 0)) + outdated = newest.get("Created", 0) > as_of.get("Created", 0) + + if as_of is None: + print_info(f"CCDB cross-check on {path}/{timestamp}: no object existed " + f"yet at the recommended timestamp (newest is {describe_object(newest)})") else: - print_info(f"CCDB cross-check on {path}{check_timestamp}: as-of object is " - f"{describe_object(result['as_of'])}") - if result["outdated"]: + print_info(f"CCDB cross-check on {path}/{timestamp}: as-of object is " + f"{describe_object(as_of)}") + if outdated: print_info(f"CCDB's current object for this window is newer " - f"({describe_object(result['newest'])}) - it postdates this pass's processing and " + f"({describe_object(newest)}) - it postdates this pass's processing and " f"was not what this run actually saw.") def describe_object(o): - return (f"Created {human(o['Created'])}, JIRA={o.get('JIRA', '?')}, " + return (f"Created {human(o.get('Created', 0))}, JIRA={o.get('JIRA', '?')}, " f"comment=\"{o.get('comment', '')}\"") From 0a11ae4cf7e809196df04c234328ce49c9ec5da7 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 07/10] getCCDBTimeMachineTimestamp.py: make CCDB cross-check non-fatal --- GRID/utils/getCCDBTimeMachineTimestamp.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index 3dacd7cdd..e6e5fc388 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -113,30 +113,35 @@ def get_ccdb_objects(url, timeout = 15): with urllib.request.urlopen(req, timeout=timeout) as resp: data = json.load(resp) except Exception as exc: - return {"error": str(exc)} + print_warning(f"Request to {url} failed: {exc}") + return None - objects = data.get("objects", []) - return objects + return data.get("objects", []) if isinstance(data, dict) else None def get_run_start_tf(run): run_info_url = f"{CCDB_HOST}/browse/RCT/Info/RunInformation/{run}" objects = get_ccdb_objects(run_info_url) - return objects[0]["STF"] + return objects[0].get("STF", 0) if objects else None def ccdb_cross_check(run, path, not_after): """Report which object a correctly time-machine-pinned query at not_after would actually resolve to, and flag if CCDB has since been updated for the - same validity window. Never raises; returns None on any failure.""" + same validity window. Never raises.""" timestamp = get_run_start_tf(run) + if not timestamp: + print_warning(f"CCDB cross-check on {path} skipped: " + "failed to obtain run start timestamp") + return + url = f"{CCDB_HOST}/browse/{path}/{timestamp}" objects = get_ccdb_objects(url) if not objects: print_warning(f"CCDB cross-check on {path}/{timestamp} skipped: " - "no CCDB object covers this instant") + f"{'CCDB request failed' if objects is None else 'no CCDB object covers this instant'}") return as_of_candidates = [o for o in objects if o.get("Created", 0) <= not_after] @@ -363,7 +368,10 @@ def main(): args.samples, args.ccdb_url, args.username, args.force_overwrite, args.publish_to_ccdb, args.skip_ccdb_query) if not args.no_ccdb_check: - ccdb_cross_check(args.run, args.ccdb_check_path, not_after) + try: + ccdb_cross_check(args.run, args.ccdb_check_path, not_after) + except Exception as exc: + print_warning(f"CCDB cross-check failed with unforeseen exception: {exc}") print(not_after) From 3453c0476938f777b20d4f23f9ec9a4ee5590835 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 08/10] getCCDBTimeMachineTimestamp.py: catch requests error in query and upload --- GRID/utils/getCCDBTimeMachineTimestamp.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index e6e5fc388..aa1ca0e63 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -236,7 +236,12 @@ def query_CCDBTimeMachineTS(base_url, user, period, passname): print_info(f"Checking if CCDBTimeMachineTS already exists: {url}") - response = requests.get(url, cert=(cert_path, key_path), verify=False) + try: + response = requests.get(url, cert=(cert_path, key_path), verify=False, timeout=15) + except (requests.RequestException, OSError) as exc: + print_warning(f"CCDB query failed with {type(exc).__name__}: {exc}") + return None, False + if response.status_code == 200: meta = extract_metadata_blocks_from_CCDB(response.content.decode('utf-8')) if (len(meta) > 0): @@ -278,14 +283,18 @@ def upload_CCDBTimeMachineTS(base_url, user, keys): print_info(f"Uploading CCDBTimeMachineTS: {url}") # Prepare request - with open(empty_file, 'rb') as f: - files = {'blob': f} - response = requests.post(url, files=files, cert=(cert_path, key_path), verify=False) - - os.remove(empty_file) + try: + with open(empty_file, 'rb') as f: + files = {'blob': f} + response = requests.post(url, files=files, cert=(cert_path, key_path), verify=False, timeout=15) + except (requests.RequestException, OSError) as exc: + print_warning(f"CCDB upload failed with {type(exc).__name__}: {exc}") + return + finally: + os.remove(empty_file) if response.status_code != 201: - print_warning(f"Upload failed, response: {response.status_code}") + print_warning(f"CCDB upload failed, response: {response.status_code}") def get_CCDBTimeMachineTS(production_tag, year, period, run, passname, From 9849b943e4e576b8cdf044b618865d5289ff000a Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 09/10] getCCDBTimeMachineTimestamp.py: set aliprod as default user in the CCDB object path --- GRID/utils/getCCDBTimeMachineTimestamp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index aa1ca0e63..3a2928819 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -364,12 +364,12 @@ def main(): parser.add_argument("--force-overwrite", action="store_true", help="overwrite the current CCDBTimeMachineTS object in the CCDB if one exists") parser.add_argument("--ccdb-url", default="https://alice-ccdb.cern.ch", - help="ccdb host for the upload") - parser.add_argument("--username", default=os.environ.get("JALIEN_USER"), - help="username to be used for the upload to CCDB") + help="ccdb host for the object query and upload") + parser.add_argument("--username", default="aliprod", + help="username to be used for the object's CCDB path") args = parser.parse_args() - missing = [n for n in ("production_tag", "year", "period", "run", "passname", "username") if not getattr(args, n)] + missing = [n for n in ("production_tag", "year", "period", "run", "passname") if not getattr(args, n)] if missing: parser.error("missing required value(s): " + ", ".join(missing)) From 758ecc874a6fb2b50350150de870b5be5f1a43e1 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 12 Nov 2026 12:38:31 +0100 Subject: [PATCH 10/10] CCDB time machine: only upload objects if running in a GRID job as aliprod --- MC/run/ANCHOR/anchorMC.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 679171e28..148876c6a 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -358,11 +358,13 @@ if [[ -n ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER} ]]; then # override CCDB time machine timestamp by hand, use e.g. 3385078236000 to effectively disable it ARG_CONDITION_NOT_AFTER="--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}" else - # options to set CCDB url, disable upload and force overwrite of an existing timestamp; default to production CCDB, uploading the object and not overwriting + # options to set CCDB url and force overwrite of an existing timestamp; default to production CCDB and not overwriting : ${CCDBTIMEMACHINE_CCDB_URL:=https://alice-ccdb.cern.ch} - [[ ${CCDBTIMEMACHINE_SKIP_UPLOAD:-} != "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --publish-to-ccdb" [[ ${CCDBTIMEMACHINE_FORCE_OVERWRITE:-} == "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --force-overwrite" + # only upload new objects if running a GRID job as aliprod + [[ ${JALIEN_USER:-} == "aliprod" ]] && [[ ${CCDBTIMEMACHINE_SKIP_UPLOAD:-} != "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --publish-to-ccdb" + # obtain CCDB time machine timestamp from CCDB or, if not available, estimate it from the creation time of the reconstruction output files CCDBTIMEMACHINE_CMD="${O2DPG_ROOT}/GRID/utils/getCCDBTimeMachineTimestamp.py --ccdb-url $CCDBTIMEMACHINE_CCDB_URL ${CCDBTIMEMACHINE_OPTIONS:-}" echo "Running command to obtain CCDB time machine timestamp: ${CCDBTIMEMACHINE_CMD}"