From 8673057ba491b3dcbbdd984fdcbd00aa55550bcf Mon Sep 17 00:00:00 2001 From: pillot Date: Fri, 21 Feb 2025 17:52:47 +0100 Subject: [PATCH] new executable to scan HV/LV CCDB objects and look for issues --- Detectors/MUON/MCH/Conditions/CMakeLists.txt | 12 + Detectors/MUON/MCH/Conditions/README.md | 53 +- .../MCH/Conditions/src/scan-hvlv-ccdb.cxx | 1121 +++++++++++++++++ 3 files changed, 1184 insertions(+), 2 deletions(-) create mode 100644 Detectors/MUON/MCH/Conditions/src/scan-hvlv-ccdb.cxx diff --git a/Detectors/MUON/MCH/Conditions/CMakeLists.txt b/Detectors/MUON/MCH/Conditions/CMakeLists.txt index f0a84dec6a416..eff44df743df5 100644 --- a/Detectors/MUON/MCH/Conditions/CMakeLists.txt +++ b/Detectors/MUON/MCH/Conditions/CMakeLists.txt @@ -37,6 +37,18 @@ o2_add_executable( O2::MCHGlobalMapping ) +o2_add_executable( + scan-hvlv-ccdb + COMPONENT_NAME mch + SOURCES src/scan-hvlv-ccdb.cxx + PUBLIC_LINK_LIBRARIES + O2::CCDB + O2::CommonUtils + O2::DetectorsDCS + O2::MCHConditions + O2::MCHStatus + ) + if(BUILD_TESTING) o2_add_test( dcs-aliases diff --git a/Detectors/MUON/MCH/Conditions/README.md b/Detectors/MUON/MCH/Conditions/README.md index fda90ba8c9ca9..d35fdcd0a0958 100644 --- a/Detectors/MUON/MCH/Conditions/README.md +++ b/Detectors/MUON/MCH/Conditions/README.md @@ -17,7 +17,9 @@ Those objects are stored at the following CCDB paths : - MCH/Calib/BadChannel - MCH/Calib/RejectList -The BadChannel and RejectList objects can be uploaded, e.g. for debug purposes, using the `o2-mch-bad-channels-ccdb` program : +# o2-mch-bad-channels-ccdb + +The BadChannel and RejectList objects can be uploaded, e.g. for debug purposes, using the `o2-mch-bad-channels-ccdb` program: ```shell $ o2-mch-bad-channels-ccdb --help @@ -46,10 +48,57 @@ Usage: -a [ --alias ] arg DCS alias (HV or LV) to reject ``` -For instance, to create in a local CCDB a RejectList object which declares solar number 32 as bad, from Tuesday 1 November 2022 00:00:01 UTC to Saturday 31 December 2022 23:59:59, use : +For instance, to create in a local CCDB a RejectList object which declares solar number 32 as bad, from Tuesday 1 November 2022 00:00:01 UTC to Saturday 31 December 2022 23:59:59, use: ```shell $ o2-mch-bad-channels-ccdb -p -s 32 -t RejectList --starttimestamp 1667260801000 --endtimestamp 1672531199000 ``` The program will search the reference CCDB (defined with `--referenceccdb`) for existing objects valid during this period and propose you to either overwrite them or update them. In the first case, a single object will be created, valid for the whole period, containing only the new bad channels. In the second case, as many objects as necessary will be created with appropriate validity ranges, adding the new bad channels to the existing ones. + +# o2-mch-scan-hvlv-ccdb + +the HV or LV DCS datapoints stored in the CCDB (http://alice-ccdb.cern.ch) can be scanned using the `o2-mch-scan-hvlv-ccdb` program: + +```shell +$ o2-mch-scan-hvlv-ccdb -h +This program scans HV or LV channels looking for issues +Usage: + -h [ --help ] produce help message + -r [ --runs ] arg run(s) to scan (comma separated list of runs + or ASCII file with one run per line) + -c [ --channels ] arg channel(s) to scan ("HV" or "LV" or comma + separated list of (part of) DCS aliases) + --configKeyValues arg Semicolon separated key=value strings to + change HV thresholds + -d [ --duration ] arg (=0) minimum duration (ms) of HV/LV issues to + consider + -w [ --warning ] arg (=1) warning level (0, 1 or 2) + -p [ --print ] arg (=1) print level (0, 1, 2 or 3) + -o [ --output ] arg (=scan.root) output root file name +``` + +It takes as input a list of runs and a list of either HV or LV channels to scan. **Note that it will scan the CCDB from the begining of the first run to the end of the last one, which can represent quite of lot of files.** More details about the options are given below. + +It produces as output a list of detected issues, with time, duration and affected runs, and a root file with the displays of the data points per channel per chamber for a visual inspection. Issues are triggered when HV/LV values go below a given threshold. For HV channels it also compares the issues found by the internal algorithm with the ones found by [Detectors/MUON/MCH/Status/src/HVStatusCreator.cxx](../Status/src/HVStatusCreator.cxx). + +For instance, to scan all HV channels for runs 545222 and 545223 and detect issues of a minimum duration of 10s, use: +```shell +o2-mch-scan-hvlv-ccdb -r 545222,545223 -c HV -d 10000 +``` + +### channel input formats: +* "HV" to scan all HV channels +* "LV" to scan all LV channels +* comma separated list of (part of) DCS aliases, which must be all of the same type, i.e contain either Quad/Slat (type = HV), or Group/an/di/Sol (type = LV) + +### warning levels: +* 0: no warning +* 1: check data points timestamp w.r.t. HV/LV file validity range with ±5s tolerance +* 2: check data points timestamp w.r.t. HV/LV file validity range without tolerance + +### print levels: +* 0: print detected issues +* 1: same as 0 + print validity range of runs and HV/LV files +* 2: same as 1 + print the first and last data points of each selected channel +* 3: same as 1 + print all the data points of each selected channel diff --git a/Detectors/MUON/MCH/Conditions/src/scan-hvlv-ccdb.cxx b/Detectors/MUON/MCH/Conditions/src/scan-hvlv-ccdb.cxx new file mode 100644 index 0000000000000..32cd365916c63 --- /dev/null +++ b/Detectors/MUON/MCH/Conditions/src/scan-hvlv-ccdb.cxx @@ -0,0 +1,1121 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "TFile.h" +#include "TCanvas.h" +#include "TGraph.h" +#include "TH1F.h" +#include "TLine.h" +#include "TMultiGraph.h" +#include "TStyle.h" + +#include "CCDB/BasicCCDBManager.h" +#include "CCDB/CcdbApi.h" +#include "CommonUtils/ConfigurableParam.h" +#include "DetectorsDCS/DataPointIdentifier.h" +#include "DetectorsDCS/DataPointValue.h" +#include "MCHConditions/DCSAliases.h" +#include "MCHStatus/HVStatusCreator.h" +#include "MCHStatus/StatusMapCreatorParam.h" + +namespace po = boost::program_options; + +using namespace o2; +using DPID = dcs::DataPointIdentifier; +using DPVAL = dcs::DataPointValue; +using DPMAP = std::unordered_map>; +using DPMAP2 = std::map>; +using RBMAP = std::map>; +using DPBMAP = std::map; +using ISSUE = std::tuple; +using ISSUELIST = std::vector; +using ISSUEMAP = std::map; + +//---------------------------------------------------------------------------- +bool containsAKey(std::string data, const std::set& Keys) +{ + /// check if the data contains one of the keys + + auto itKey = std::find_if(Keys.begin(), Keys.end(), [&data](const auto& key) { + return data.find(key) != data.npos; + }); + + return itKey != Keys.end(); +} + +//---------------------------------------------------------------------------- +bool isValid(std::string alias) +{ + /// check if the alias is a valid (part of a) DCS alias + + static const std::vector aliases = + mch::dcs::aliases({mch::dcs::MeasurementType::HV_V, + mch::dcs::MeasurementType::LV_V_FEE_ANALOG, + mch::dcs::MeasurementType::LV_V_FEE_DIGITAL, + mch::dcs::MeasurementType::LV_V_SOLAR}); + + auto itAlias = std::find_if(aliases.begin(), aliases.end(), [&alias](const auto& a) { + return a.find(alias) != a.npos; + }); + + return itAlias != aliases.end(); +} + +//---------------------------------------------------------------------------- +void scanWhat(std::string what, std::string& path, bool& scanHV, bool& scanAll, std::set& aliases) +{ + /// get what to scan and where + + static const std::set hvKeys{"HV", "Quad", "Slat"}; + static const std::set lvKeys{"LV", "Group", "an", "di", "Sol"}; + + // HV or LV ? + path = ""; + scanHV = false; + if (containsAKey(what, hvKeys)) { + path = "MCH/Calib/HV"; + scanHV = true; + } + if (containsAKey(what, lvKeys)) { + if (scanHV) { + printf("error: cannot scan HV and LV channels at the same time\n"); + exit(1); + } + path = "MCH/Calib/LV"; + } + if (path.empty()) { + printf("error: no valid HV or LV channel to scan\n"); + exit(1); + } + + // everything or specific aliases ? + if (what.find(scanHV ? "HV" : "LV") != what.npos) { + scanAll = true; + aliases.clear(); + } else { + scanAll = false; + std::istringstream input(what); + for (std::string alias; std::getline(input, alias, ',');) { + if (isValid(alias)) { + aliases.insert(alias); + } else { + printf("error: \"%s\" invalid (part of) HV or LV alias\n", alias.c_str()); + exit(1); + } + } + } +} + +//---------------------------------------------------------------------------- +uint64_t ms2s(uint64_t ts) +{ + /// convert the time stamp from ms to s + + return (ts + 500) / 1000; +} + +//---------------------------------------------------------------------------- +std::string getTime(uint64_t ts) +{ + /// convert the time stamp (ms) to local time + + time_t t = ms2s(ts); + + std::string time = std::ctime(&t); + time.pop_back(); // remove trailing \n + + return time; +} + +//---------------------------------------------------------------------------- +std::set getRuns(std::string runList) +{ + /// read the runList from an ASCII file, or a comma separated run list, or a single run + + std::set runs{}; + + auto isNumber = [](std::string val) { return !val.empty() && val.find_first_not_of("0123456789") == val.npos; }; + + if (isNumber(runList)) { + + runs.insert(std::stoi(runList)); + + } else if (runList.find(",") != runList.npos) { + + std::istringstream input(runList); + for (std::string run; std::getline(input, run, ',');) { + if (isNumber(run)) { + runs.insert(std::stoi(run)); + } + } + + } else { + + std::ifstream input(runList); + if (input.is_open()) { + for (std::string run; std::getline(input, run);) { + if (isNumber(run)) { + runs.insert(std::stoi(run)); + } + } + } + } + + return runs; +} + +//---------------------------------------------------------------------------- +RBMAP getRunBoundaries(ccdb::CcdbApi const& api, std::string runList) +{ + /// return the SOR / EOR time stamps for every runs in the list + + RBMAP runBoundaries{}; + + auto runs = getRuns(runList); + + for (auto run : runs) { + auto boundaries = ccdb::CCDBManagerInstance::getRunDuration(api, run); + runBoundaries.emplace(run, boundaries); + } + + return runBoundaries; +} + +//---------------------------------------------------------------------------- +void checkRunBoundaries(const RBMAP& runBoundaries) +{ + /// check the consistency of the run time boundaries + + if (runBoundaries.empty()) { + printf("error: no run found from the list\n"); + exit(1); + } + + bool error = false; + int previousRun = 0; + uint64_t endOfPreviousRun = 0; + + for (const auto& [run, boundaries] : runBoundaries) { + if (boundaries.second <= boundaries.first) { + printf("error: run %d EOR <= SOR: %llu - %llu (%s - %s)\n", + run, boundaries.first, boundaries.second, + getTime(boundaries.first).c_str(), getTime(boundaries.second).c_str()); + error = true; + } + if (boundaries.first <= endOfPreviousRun) { + printf("error: SOR run %d <= EOR run %d: %llu (%s) <= %llu (%s)\n", + run, previousRun, boundaries.first, getTime(boundaries.first).c_str(), + endOfPreviousRun, getTime(endOfPreviousRun).c_str()); + error = true; + } + previousRun = run; + endOfPreviousRun = boundaries.second; + } + + if (error) { + exit(1); + } +} + +//---------------------------------------------------------------------------- +void printRunBoundaries(const RBMAP& runBoundaries) +{ + /// print the list of runs with their time boundaries + + printf("\nlist of runs with their boundaries:\n"); + printf("------------------------------------\n"); + + for (const auto& [run, boundaries] : runBoundaries) { + printf("%d: %llu - %llu (%s - %s)\n", run, boundaries.first, boundaries.second, + getTime(boundaries.first).c_str(), getTime(boundaries.second).c_str()); + } + + printf("------------------------------------\n"); +} + +//---------------------------------------------------------------------------- +void drawRunBoudaries(const RBMAP& runBoundaries, TCanvas* c) +{ + /// draw the run time boundaries + + c->cd(); + + for (const auto& [run, boundaries] : runBoundaries) { + + TLine* startRunLine = new TLine(ms2s(boundaries.first), c->GetUymin(), ms2s(boundaries.first), c->GetUymax()); + startRunLine->SetUniqueID(run); + startRunLine->SetLineColor(4); + startRunLine->SetLineWidth(1); + startRunLine->Draw(); + + TLine* endRunLine = new TLine(ms2s(boundaries.second), c->GetUymin(), ms2s(boundaries.second), c->GetUymax()); + endRunLine->SetUniqueID(run); + endRunLine->SetLineColor(2); + endRunLine->SetLineWidth(1); + endRunLine->Draw(); + } +} + +//---------------------------------------------------------------------------- +DPBMAP getDPBoundaries(ccdb::CcdbApi const& api, std::string what, uint64_t tStart, uint64_t tStop) +{ + /// get the time boundaries of every HV/LV files found in the time range + + // add extra margin (ms) of ± 1 min to the creation time, which occurs every 30 min + static const uint64_t timeMarging[2] = {60000, 1860000}; + + std::istringstream fileInfo(api.list(what.c_str(), false, "text/plain", + tStop + timeMarging[1], tStart - timeMarging[0])); + + DPBMAP dpBoundaries{}; + std::string dummy{}; + uint64_t begin = 0; + uint64_t end = 0; + + for (std::string line; std::getline(fileInfo, line);) { + if (line.find("Validity:") == 0) { + std::istringstream in(line); + in >> dummy >> begin >> dummy >> end; + dpBoundaries.emplace(begin, end); + } + } + + if (dpBoundaries.empty()) { + printf("\e[0;31merror: no file found in %s in time range %llu - %llu (%s - %s) --> use the default one\e[0m\n", + what.c_str(), tStart, tStop, getTime(tStart).c_str(), getTime(tStop).c_str()); + dpBoundaries.emplace(1, 9999999999999); + } + + return dpBoundaries; +} + +//---------------------------------------------------------------------------- +void checkDPBoundaries(const DPBMAP& dpBoundaries, bool scanHV, uint64_t tStart, uint64_t tStop) +{ + /// check the consistency of HV/LV file time boundaries + + bool error = false; + + if (dpBoundaries.begin()->first > tStart) { + printf("error: the beginning of the time range is not covered: %llu > %llu (%s > %s)\n", + dpBoundaries.begin()->first, tStart, + getTime(dpBoundaries.begin()->first).c_str(), getTime(tStart).c_str()); + error = true; + } + if (dpBoundaries.rbegin()->second < tStop) { + printf("error: the end of the time range is not covered: %llu < %llu (%s < %s)\n", + dpBoundaries.rbegin()->second, tStop, + getTime(dpBoundaries.rbegin()->second).c_str(), getTime(tStop).c_str()); + error = true; + } + + uint64_t previousTStop = dpBoundaries.begin()->first; + for (auto [tStart, tStop] : dpBoundaries) { + if (tStop <= tStart) { + printf("error: EOF <= SOF: %llu - %llu (%s - %s)\n", + tStart, tStop, getTime(tStart).c_str(), getTime(tStop).c_str()); + error = true; + } + if (tStart != previousTStop) { + printf("error: end of %s file != start of next %s file: %llu (%s) != %llu (%s))\n", + scanHV ? "HV" : "LV", scanHV ? "HV" : "LV", + previousTStop, getTime(previousTStop).c_str(), tStart, getTime(tStart).c_str()); + error = true; + } + previousTStop = tStop; + } + + if (error) { + exit(1); + } +} + +//---------------------------------------------------------------------------- +void printDPBoundaries(const DPBMAP& dpBoundaries, bool scanHV) +{ + /// print the time boundaries of every HV/LV files found in the full time range + + printf("\nlist of %s file time boundaries:\n", scanHV ? "HV" : "LV"); + printf("------------------------------------\n"); + + for (auto [tStart, tStop] : dpBoundaries) { + printf("%llu - %llu (%s - %s)\n", tStart, tStop, getTime(tStart).c_str(), getTime(tStop).c_str()); + } + + printf("------------------------------------\n"); +} + +//---------------------------------------------------------------------------- +double getLVLimit(std::string alias) +{ + /// return the LV limit for that channel + + static const double lvLimits[3] = {1.5, 1.5, 6.}; // FeeAnalog, FeeDigital, Solar + + if (alias.find("an") != alias.npos) { + return lvLimits[0]; + } else if (alias.find("di") != alias.npos) { + return lvLimits[1]; + } + return lvLimits[2]; +} + +//---------------------------------------------------------------------------- +void drawLimit(double limit, TCanvas* c) +{ + /// draw the HV/LV limit for the displayed chamber + + c->cd(); + + TLine* l = new TLine(c->GetUxmin(), limit, c->GetUxmax(), limit); + l->SetLineColor(1); + l->SetLineWidth(1); + l->SetLineStyle(2); + l->Draw(); +} + +//---------------------------------------------------------------------------- +std::string getDuration(uint64_t tStart, uint64_t tStop) +{ + /// get the duration (dd hh:mm:ss) between the two time stamps (ms) + + auto dt = ms2s(tStop - tStart); + auto s = dt % 60; + auto m = (dt / 60) % 60; + auto h = (dt / 3600) % 24; + auto d = dt / 86400; + + return fmt::format("{:02}d {:02}:{:02}:{:02}", d, h, m, s); +} + +//---------------------------------------------------------------------------- +double getValue(DPVAL dp) +{ + /// return the value of this data point + + union Converter { + uint64_t raw_data; + double value; + } converter; + + converter.raw_data = dp.payload_pt1; + + return converter.value; +} + +//---------------------------------------------------------------------------- +std::string getDE(std::string alias) +{ + /// for DCS HV alias: return the corresponding DE (and sector) + /// for DCS LV alias: return an empty string + + auto de = mch::dcs::aliasToDetElemId(alias); + + if (de) { + return (mch::dcs::isQuadrant(mch::dcs::aliasToChamber(alias))) + ? fmt::format("DE{}-{}", *de, mch::dcs::aliasToNumber(alias) % 10) + : fmt::format("DE{}", *de); + } + + return ""; +} + +//---------------------------------------------------------------------------- +void fillDataPoints(const std::vector& dps, std::map& dps2, + uint64_t tMin, uint64_t tMax, int warningLevel) +{ + /// fill the map of data points + + static const uint64_t tolerance = 5000; + + if (dps.empty()) { + printf("error: the file does not contain any data point\n"); + exit(1); + } + + auto itDP = dps.begin(); + auto ts = itDP->get_epoch_time(); + std::string header = "warning:"; + std::string color = (ts + tolerance < tMin || ts > tMin + tolerance) ? "\e[0;31m" : "\e[0;34m"; + bool printWarning = warningLevel > 1 || (warningLevel == 1 && color == "\e[0;31m"); + + // check if the first data point is a copy of the last one from previous file + if (!dps2.empty()) { + auto previousTS = dps2.rbegin()->first; + if (ts != previousTS || getValue(*itDP) != dps2.rbegin()->second) { + if (ts <= previousTS) { + printf("error: wrong data point order (%llu <= %llu)\n", ts, previousTS); + exit(1); + } + if (printWarning) { + printf("%s%s missing the previous data point (dt = %s%llu ms)", color.c_str(), header.c_str(), + (previousTS < tMin) ? "-" : "+", (previousTS < tMin) ? tMin - previousTS : previousTS - tMin); + if (ts <= tMin) { + printf(" but get one at dt = -%llu ms\e[0m\n", tMin - ts); + } else { + printf("\e[0m\n"); + } + header = " "; + } + } + } + + // add the first data point (should be before the start of validity of the file) + if (ts >= tMax) { + printf("error: first data point exceeding file validity range (dt = +%llu ms)\n", ts - tMax); + exit(1); + } else if (ts > tMin && printWarning) { + printf("%s%s missing data point prior file start of validity (dt = +%llu ms)\e[0m\n", + color.c_str(), header.c_str(), ts - tMin); + header = " "; + } + dps2.emplace(ts, getValue(*itDP)); + + // add other data points (should be within the validity range of the file) + auto previousTS = ts; + for (++itDP; itDP < dps.end(); ++itDP) { + ts = itDP->get_epoch_time(); + if (ts <= previousTS) { + printf("error: wrong data point order (%llu <= %llu)\n", ts, previousTS); + exit(1); + } + if (ts < tMin && (warningLevel > 1 || (warningLevel == 1 && ts + tolerance < tMin))) { + printf("%s%s data point outside of file validity range (dt = -%llu ms)\e[0m\n", + (ts + tolerance < tMin) ? "\e[0;31m" : "\e[0;34m", header.c_str(), tMin - ts); + } else if (ts >= tMax && warningLevel >= 1) { + printf("\e[0;31m%s data point outside of file validity range (dt = +%llu ms)\e[0m\n", + header.c_str(), ts - tMax); + } + dps2.emplace(ts, getValue(*itDP)); + previousTS = ts; + } +} + +//---------------------------------------------------------------------------- +void selectDataPoints(DPMAP2 dpsMapsPerCh[10], uint64_t tStart, uint64_t tStop) +{ + /// remove the data points outside of the given time range and, if needed, + /// add a data point at the boundaries with value equal to the preceding one + + for (int ch = 0; ch < 10; ++ch) { + for (auto& [alias, dps] : dpsMapsPerCh[ch]) { + + // get the first data point in the time range, remove the previous ones + // and add a data point with value equal to the preceding one if it exits + // or to this one otherwise + auto itFirst = dps.lower_bound(tStart); + if (itFirst != dps.begin()) { + double previousVal = std::prev(itFirst)->second; + for (auto it = dps.begin(); it != itFirst;) { + it = dps.erase(it); + } + dps.emplace(tStart, previousVal); + } else if (itFirst->first != tStart) { + if (itFirst->first > tStop) { + printf("error (%s): all data points are posterior to the end of the time range\n", alias.c_str()); + } else { + printf("error (%s): first data point is posterior to the beginning of the time range\n", alias.c_str()); + } + dps.emplace(tStart, itFirst->second); + } + + // get the first data point exceeding the time range, remove it and the next ones + // and add a data point with value equal to the preceding one if needed + auto itLast = dps.upper_bound(tStop); + double previousVal = std::prev(itLast)->second; + for (auto it = itLast; it != dps.end();) { + it = dps.erase(it); + } + dps.emplace(tStop, previousVal); + } + } +} + +//---------------------------------------------------------------------------- +void printDataPoints(const DPMAP2 dpsMapsPerCh[10], std::string hvlvFormat, bool all) +{ + /// print all the registered data points + + const auto format1 = fmt::format(" %llu (%s): {} V\n", hvlvFormat.c_str()); + const auto format2 = fmt::format(": %llu (%s): {} V -- %llu (%s): {} V\n", + hvlvFormat.c_str(), hvlvFormat.c_str()); + + for (int ch = 0; ch < 10; ++ch) { + + printf("\n------------ chamber %d ------------\n", ch + 1); + + for (const auto& [alias, dps] : dpsMapsPerCh[ch]) { + + printf("- %s: %lu values", alias.c_str(), dps.size()); + + if (all) { + + printf("\n"); + for (const auto& [ts, val] : dps) { + printf(format1.c_str(), ts, getTime(ts).c_str(), val); + } + + } else if (!dps.empty()) { + + const auto firstdt = dps.begin(); + const auto lastdt = dps.rbegin(); + printf(format2.c_str(), + firstdt->first, getTime(firstdt->first).c_str(), firstdt->second, + lastdt->first, getTime(lastdt->first).c_str(), lastdt->second); + + } else { + printf("\n"); + } + } + } +} + +//---------------------------------------------------------------------------- +TGraph* mapToGraph(std::string alias, const std::map& dps) +{ + /// create a graph for the DCS channel and add the data points + + TGraph* g = new TGraph(dps.size()); + + auto pos = alias.find("."); + auto shortAlias = alias.substr(0, pos); + auto de = getDE(alias); + auto title = de.empty() ? fmt::format("{}", shortAlias.c_str()) + : fmt::format("{} ({})", de.c_str(), shortAlias.c_str()); + g->SetNameTitle(alias.c_str(), title.c_str()); + + int i(0); + for (auto [ts, val] : dps) { + g->SetPoint(i, ms2s(ts), val); + ++i; + } + + g->SetMarkerSize(1.5); + g->SetMarkerStyle(2); + g->SetLineStyle(2); + + return g; +} + +//---------------------------------------------------------------------------- +TCanvas* drawDataPoints(TMultiGraph* mg, double min, double max) +{ + /// display the data points of the given chamber + + TCanvas* c = new TCanvas(mg->GetName(), mg->GetHistogram()->GetTitle(), 1500, 900); + + mg->Draw("A plc pmc"); + mg->SetMinimum(min); + mg->SetMaximum(max); + mg->GetXaxis()->SetTimeDisplay(1); + mg->GetXaxis()->SetTimeFormat("%d/%m %H:%M"); + mg->GetXaxis()->SetTimeOffset(0, "local"); + mg->GetXaxis()->SetNdivisions(21010); + + c->BuildLegend(); + c->Update(); + + return c; +} + +//---------------------------------------------------------------------------- +void findIssues(const std::map& dps, double limit, ISSUELIST& issues) +{ + /// return the list of HV/LV issues (time range, min value, mean value) for each DCS channel + + uint64_t tStart(0); + double min(0.); + double mean(0.); + uint64_t prevTS(0); + double prevVal(-1.); + + for (auto [ts, val] : dps) { + + if (val < limit) { + + if (tStart == 0) { + + // start a new issue... + tStart = ts; + min = val; + mean = 0.; + prevTS = ts; + prevVal = val; + + } else { + + // ... or complement the current one + min = std::min(min, val); + mean += prevVal * (ts - prevTS); + prevTS = ts; + prevVal = val; + } + + } else if (tStart > 0) { + + // complete the current issue, if any, and register it + mean += prevVal * (ts - prevTS); + mean /= (ts - tStart); + issues.emplace_back(tStart, ts, min, mean, ""); + tStart = 0; + } + } + + // complete the last issue, if any and its duration is != 0, and register it + if (tStart > 0 && prevTS != tStart) { + mean /= (prevTS - tStart); + issues.emplace_back(tStart, prevTS, min, mean, ""); + } +} + +//---------------------------------------------------------------------------- +void fillO2Issues(const std::vector& o2issues, ISSUELIST& issues, + uint64_t tMin, uint64_t tMax) +{ + /// fill the list of issues from O2 (extend the previous one and/or create new ones) + + // the list must not be empty + if (o2issues.empty()) { + printf("error: O2 returns an empty list of issues\n"); + exit(1); + } + + for (auto itIssue = o2issues.begin(); itIssue != o2issues.end(); ++itIssue) { + + // exclude issues fully outside of the DP file boudaries + if (itIssue->end <= tMin || itIssue->begin >= tMax) { + printf("\e[0;35mwarning: skipping O2 issue outside of file boundaries (%llu - %llu)\e[0m\n", + itIssue->begin, itIssue->end); + continue; + } + + // only the first issue could in principle extend before the start of the DP file, to O + if (itIssue->begin < tMin - mch::StatusMapCreatorParam::Instance().timeMargin && + (itIssue != o2issues.begin() || itIssue->begin != 0)) { + printf("\e[0;35mwarning: O2 returns an issue with uncommon start time (%llu < %llu)\e[0m\n", + itIssue->begin, tMin - mch::StatusMapCreatorParam::Instance().timeMargin); + } + + // only the last issue could in principle extend beyond the end of the DP file, to infinity + if (itIssue->end >= tMax + mch::StatusMapCreatorParam::Instance().timeMargin && + (itIssue != std::prev(o2issues.end()) || itIssue->end != std::numeric_limits::max())) { + printf("\e[0;35mwarning: O2 returns an issue with uncommon end time (%llu >= %llu)\e[0m\n", + itIssue->end, tMax + mch::StatusMapCreatorParam::Instance().timeMargin); + } + + // extend the last issue in case of continuity accross the DP files or add a new one, + // restricting their time range within the DP file boundaries + if (itIssue->begin <= tMin && !issues.empty() && std::get<1>(issues.back()) == tMin) { + std::get<1>(issues.back()) = std::min(itIssue->end, tMax); + } else { + issues.emplace_back(std::max(itIssue->begin, tMin), std::min(itIssue->end, tMax), 0., 0., ""); + } + } +} + +//---------------------------------------------------------------------------- +std::string findAffectedRuns(const RBMAP& runBoundaries, uint64_t tStart, uint64_t tStop) +{ + /// return the list of affected runs in this time range + + std::string runs; + + for (const auto& [run, boundaries] : runBoundaries) { + + if (boundaries.second <= tStart) { + continue; + } else if (boundaries.first >= tStop) { + break; + } + + runs += fmt::format("{},", run); + } + + if (!runs.empty()) { + runs.pop_back(); + } + + return runs; +} + +//---------------------------------------------------------------------------- +void selectIssues(ISSUEMAP issuesPerCh[10], const RBMAP& runBoundaries, uint64_t minDuration) +{ + /// select HV/LV issues of a minimum duration (ms) occurring during runs + + for (int ch = 0; ch < 10; ++ch) { + for (auto& issues : issuesPerCh[ch]) { + for (auto itIssue = issues.second.begin(); itIssue != issues.second.end();) { + + auto tStart = std::get<0>(*itIssue); + auto tStop = std::get<1>(*itIssue); + + if (tStop - tStart < minDuration) { + + itIssue = issues.second.erase(itIssue); + + } else { + + auto runs = findAffectedRuns(runBoundaries, tStart, tStop); + + if (runs.empty()) { + + itIssue = issues.second.erase(itIssue); + + } else { + + std::get<4>(*itIssue) = runs; + ++itIssue; + } + } + } + } + } +} + +//---------------------------------------------------------------------------- +void selectO2Issues(ISSUEMAP issuesPerCh[10], const RBMAP& runBoundaries) +{ + /// select HV issues from O2 algorithm occurring during runs + /// and restrict the range of issues to the run range + + for (int ch = 0; ch < 10; ++ch) { + for (auto& issues : issuesPerCh[ch]) { + for (auto itIssue = issues.second.begin(); itIssue != issues.second.end();) { + + auto& tStart = std::get<0>(*itIssue); + auto& tStop = std::get<1>(*itIssue); + + auto runs = findAffectedRuns(runBoundaries, tStart, tStop); + + if (runs.empty()) { + + itIssue = issues.second.erase(itIssue); + + } else { + + tStart = std::max(tStart, runBoundaries.begin()->second.first); + tStop = std::min(tStop, runBoundaries.rbegin()->second.second); + std::get<4>(*itIssue) = runs; + ++itIssue; + } + } + } + } +} + +//---------------------------------------------------------------------------- +bool eraseIssue(const ISSUE& issue, ISSUELIST& issues) +{ + /// find an issue with the same time range and associated run list and erase it + /// return true in case of success + + auto itIssue = std::find_if(issues.begin(), issues.end(), [&issue](const auto& i) { + return (std::get<0>(i) == std::get<0>(issue) && + std::get<1>(i) == std::get<1>(issue) && + std::get<4>(i) == std::get<4>(issue)); + }); + + if (itIssue != issues.end()) { + issues.erase(itIssue); + return true; + } + + return false; +} + +//---------------------------------------------------------------------------- +void printIssues(const ISSUEMAP issuesPerCh[10], const ISSUEMAP o2IssuesPerCh[10], + bool scanHV, std::string hvlvFormat) +{ + /// print all HV/LV issues + + // copy the issues so that we can modify them (i.e. add empty lists or delete issues after printing) + ISSUEMAP issuesPerChCopy[10]; + ISSUEMAP o2IssuesPerChCopy[10]; + for (int ch = 0; ch < 10; ++ch) { + issuesPerChCopy[ch] = issuesPerCh[ch]; + o2IssuesPerChCopy[ch] = o2IssuesPerCh[ch]; + } + + // make sure that all alias keys in the map o2IssuesPerChCopy are also in issuesPerChCopy in order to + // simplify the loop over all issues from both algorithms and fix the order in which they are printed + for (int ch = 0; ch < 10; ++ch) { + for (const auto& [alias, o2Issues] : o2IssuesPerChCopy[ch]) { + if (!o2Issues.empty()) { + issuesPerChCopy[ch].try_emplace(alias, ISSUELIST{}); + } + } + } + + auto printHeader = [](std::string alias) { + auto de = getDE(alias); + if (de.empty()) { + printf("Problem found for %s:\n", alias.c_str()); + } else { + printf("Problem found for %s (%s):\n", alias.c_str(), de.c_str()); + } + }; + + const auto format = fmt::format("%llu - %llu: %s (duration = %s, min = {} V, mean = {} V) --> run(s) %s\n", + hvlvFormat.c_str(), hvlvFormat.c_str()); + + auto printIssue = [&format](ISSUE issue, std::string color) { + const auto& [tStart, tStop, min, mean, runs] = issue; + printf("%s", color.c_str()); + printf(format.c_str(), tStart, tStop, + getTime(tStart).c_str(), getDuration(tStart, tStop).c_str(), min, mean, runs.c_str()); + printf("\e[0m"); + }; + + if (scanHV) { + printf("\n------ list of issues from \e[0;31mthis macro only\e[0m, \e[0;35mO2 only\e[0m, or \e[0;32mboth\e[0m ------\n"); + } else { + printf("\n------ list of issues ------\n"); + } + + bool foundIssues = false; + + for (int ch = 0; ch < 10; ++ch) { + for (const auto& [alias, issues] : issuesPerChCopy[ch]) { + + auto& o2Issues = o2IssuesPerChCopy[ch][alias]; + + if (!issues.empty() || !o2Issues.empty()) { + + foundIssues = true; + printHeader(alias); + + // print all issues found by this macro + for (const auto& issue : issues) { + // change color if the issue is not found by the O2 algorithm (only for HV) + std::string color = (scanHV && !eraseIssue(issue, o2Issues)) ? "\e[0;31m" : "\e[0;32m"; + printIssue(issue, color); + } + + // print other issues found by the O2 algorithm + for (const auto& issue : o2Issues) { + printIssue(issue, "\e[0;35m"); + } + + printf("----------------------------\n"); + } + } + } + + if (!foundIssues) { + printf("----------------------------\n"); + } +} + +//---------------------------------------------------------------------------- +int main(int argc, char** argv) +{ + /// scan HV or LV CCDB objects looking for issues + + std::string runList = ""; + std::string what = ""; + std::string config = ""; + uint64_t minDuration = 0; + int warningLevel = 1; + int printLevel = 1; + std::string outFileName = ""; + + po::options_description usage("Usage"); + // clang-format off + usage.add_options() + ("help,h", "produce help message") + ("runs,r",po::value(&runList)->default_value(""),"run(s) to scan (comma separated list of runs or ASCII file with one run per line)") + ("channels,c",po::value(&what)->default_value(""),R"(channel(s) to scan ("HV" or "LV" or comma separated list of (part of) DCS aliases))") + ("configKeyValues",po::value(&config)->default_value(""),"Semicolon separated key=value strings to change HV thresholds") + ("duration,d",po::value(&minDuration)->default_value(0),"minimum duration (ms) of HV/LV issues to consider") + ("warning,w",po::value(&warningLevel)->default_value(1),"warning level (0, 1 or 2)") + ("print,p",po::value(&printLevel)->default_value(1),"print level (0, 1, 2 or 3)") + ("output,o",po::value(&outFileName)->default_value("scan.root"),"output root file name") + ; + // clang-format on + + po::options_description cmdline; + cmdline.add(usage); + + po::variables_map vm; + po::store(po::command_line_parser(argc, argv).options(cmdline).run(), vm); + + if (vm.count("help")) { + std::cout << "This program scans HV or LV channels looking for issues\n"; + std::cout << usage << "\n"; + return 2; + } + + try { + po::notify(vm); + } catch (const po::error& e) { + std::cout << "error: " << e.what() << "\n"; + exit(1); + } + + if (runList.empty()) { + printf("error: you must provide run(s) to scan\n"); + exit(1); + } + + if (what.empty()) { + printf("error: you must provide channel(s) to scan\n"); + exit(1); + } + + // setup printout and display + const double hvRange[2] = {-10., 1700.}; + const double lvRange[3] = {-1., 4., 8.}; // min, max FeeAnalog/FeeDigital, max Solar + const std::string hvFormat = "%7.2f"; + const std::string lvFormat = "%4.2f"; + gStyle->SetPalette(kVisibleSpectrum); + + // setup algorithms searching for HV issues + conf::ConfigurableParam::updateFromString(config); + conf::ConfigurableParam::setValue("MCHStatusMap.hvMinDuration", std::to_string(minDuration)); + conf::ConfigurableParam::setValue("MCHStatusMap.timeMargin", "0"); // must be 0 to compare O2 with this scan + + // determine what is scanned + std::string path{}; + bool scanHV = false; + bool scanAll = false; + std::set aliases{}; + scanWhat(what, path, scanHV, scanAll, aliases); + + ccdb::CcdbApi api; + api.init("http://alice-ccdb.cern.ch"); + + // get the SOR/EOR of every runs from the list, ordered in run number + auto runBoundaries = getRunBoundaries(api, runList); + if (printLevel > 0) { + printRunBoundaries(runBoundaries); + } + checkRunBoundaries(runBoundaries); + + // extract the time boundaries for each HV/LV file in the full time range + auto dpBoundaries = getDPBoundaries(api, path.c_str(), runBoundaries.begin()->second.first, + runBoundaries.rbegin()->second.second); + if (printLevel > 0) { + printDPBoundaries(dpBoundaries, scanHV); + } + checkDPBoundaries(dpBoundaries, scanHV, runBoundaries.begin()->second.first, + runBoundaries.rbegin()->second.second); + + // loop over the HV/LV files, fill the lists of data points per chamber and find issues using O2 algorithm + DPMAP2 dpsMapsPerCh[10]; + mch::HVStatusCreator hvStatusCreator{}; + ISSUEMAP o2issuesPerCh[10]; + std::map metadata; + for (auto boundaries : dpBoundaries) { + + auto* dpMap = api.retrieveFromTFileAny(path.c_str(), metadata, boundaries.first); + + // fill the lists of data points per chamber for requested aliases + for (const auto& [dpid, dps] : *dpMap) { + std::string alias(dpid.get_alias()); + if (!mch::dcs::isValid(alias)) { + printf("error: invalid DCS alias: %s\n", alias.c_str()); + exit(1); + } + if ((scanAll || containsAKey(alias, aliases)) && (!scanHV || alias.find(".iMon") == alias.npos)) { + int chamber = mch::dcs::toInt(mch::dcs::aliasToChamber(alias)); + fillDataPoints(dps, dpsMapsPerCh[chamber][alias], boundaries.first, boundaries.second, warningLevel); + } + } + + // find issues for requested aliases using O2 algorithm (only for HV) + if (scanHV) { + hvStatusCreator.findBadHVs(*dpMap); + for (const auto& [alias, issues] : hvStatusCreator.getBadHVs()) { + if (scanAll || containsAKey(alias, aliases)) { + int chamber = mch::dcs::toInt(mch::dcs::aliasToChamber(alias)); + fillO2Issues(issues, o2issuesPerCh[chamber][alias], boundaries.first, boundaries.second); + } + } + } + } + if (printLevel > 1) { + printf("\nall data points:"); + printDataPoints(dpsMapsPerCh, scanHV ? hvFormat : lvFormat, printLevel > 2); + } + + // select the data points in the time range + selectDataPoints(dpsMapsPerCh, runBoundaries.begin()->second.first, runBoundaries.rbegin()->second.second); + if (printLevel > 1) { + printf("\ndata points in the time range covered by runs:"); + printDataPoints(dpsMapsPerCh, scanHV ? hvFormat : lvFormat, printLevel > 2); + } + + // create and fill the graphs, and find HV/LV issues + ISSUEMAP issuesPerCh[10]; + TMultiGraph* mg[10]; + std::set limits; + for (int ch = 0; ch < 10; ++ch) { + mg[ch] = new TMultiGraph; + mg[ch]->SetNameTitle(fmt::format("ch{}", ch + 1).c_str(), + fmt::format("chamber {};time;{} (V)", ch + 1, scanHV ? "HV" : "LV").c_str()); + for (const auto& [alias, dps] : dpsMapsPerCh[ch]) { + mg[ch]->Add(mapToGraph(alias, dps), "lp"); + auto limit = scanHV ? mch::StatusMapCreatorParam::Instance().hvLimits[ch] : getLVLimit(alias); + limits.emplace(limit); + findIssues(dps, limit, issuesPerCh[ch][alias]); + } + } + + // select HV/LV issues of a minimum duration (ms) occurring during runs + selectIssues(issuesPerCh, runBoundaries, minDuration); + selectO2Issues(o2issuesPerCh, runBoundaries); + printIssues(issuesPerCh, o2issuesPerCh, scanHV, scanHV ? hvFormat : lvFormat); + + // display + TCanvas* c[10]; + for (int ch = 0; ch < 10; ++ch) { + if (scanHV) { + c[ch] = drawDataPoints(mg[ch], hvRange[0], hvRange[1]); + drawLimit(mch::StatusMapCreatorParam::Instance().hvLimits[ch], c[ch]); + } else { + auto lvMax = (what.find("LV") != what.npos || what.find("Sol") != what.npos) ? lvRange[2] : lvRange[1]; + c[ch] = drawDataPoints(mg[ch], lvRange[0], lvMax); + for (auto limit : limits) { + drawLimit(limit, c[ch]); + } + } + drawRunBoudaries(runBoundaries, c[ch]); + } + + // save display + TFile dataFile(outFileName.c_str(), "recreate"); + for (int ch = 0; ch < 10; ++ch) { + c[ch]->Write(); + } + dataFile.Close(); + + return 0; +}