diff --git a/CMakeLists.txt b/CMakeLists.txt index 11df14a6b8..93d17d8090 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ add_subdirectory(CosmicReco) add_subdirectory(CRVConditions) add_subdirectory(CRVConfig) add_subdirectory(CRVFilters) +add_subdirectory(DQMHelpers) add_subdirectory(CRVReco) add_subdirectory(CRVResponse) add_subdirectory(CRVTools) diff --git a/CRVReco/CMakeLists.txt b/CRVReco/CMakeLists.txt index d5424624ec..40ea50198f 100644 --- a/CRVReco/CMakeLists.txt +++ b/CRVReco/CMakeLists.txt @@ -65,6 +65,7 @@ cet_build_plugin(CrvDQMcollector art::module LIBRARIES REG art_root_io::TFileService_service Offline::CRVReco + Offline::DQMHelpers Offline::CosmicRayShieldGeom Offline::CRVConditions diff --git a/CRVReco/fcl/prolog_v12.fcl b/CRVReco/fcl/prolog_v12.fcl index 28ecc1fe69..67b3714361 100644 --- a/CRVReco/fcl/prolog_v12.fcl +++ b/CRVReco/fcl/prolog_v12.fcl @@ -26,9 +26,13 @@ BEGIN_PROLOG module_type : CrvDQMcollector useDQMcollector : true crvDigiModuleLabel : "CrvDigi" + crvStatusModuleLabel : "CrvDigi" #crvRecoPulsesModuleLabel : "CrvRecoPulses" crvCoincidenceClusterFinderModuleLabel : "CrvCoincidenceClusterFinder" crvDaqErrorModuleLabel : "CrvDigi" + crvDigiDQMDir : "CRVDigiDQM" + fillInclusiveDigiDQM : true + crvDigiDQMkppReadout : true # KPP FEB-axis sizing (ROC 1-2) } CrvRecoPulses: diff --git a/CRVReco/src/CrvDQMcollector_module.cc b/CRVReco/src/CrvDQMcollector_module.cc index f43e778b6b..c8040595d7 100644 --- a/CRVReco/src/CrvDQMcollector_module.cc +++ b/CRVReco/src/CrvDQMcollector_module.cc @@ -13,7 +13,9 @@ #include "Offline/GeometryService/inc/GeomHandle.hh" #include "Offline/GeometryService/inc/GeometryService.hh" #include "Offline/ProditionsService/inc/ProditionsHandle.hh" +#include "Offline/DQMHelpers/inc/CRVDigiDQM.hh" #include "Offline/RecoDataProducts/inc/CrvDigi.hh" +#include "Offline/RecoDataProducts/inc/CrvStatus.hh" #include "Offline/RecoDataProducts/inc/CrvRecoPulse.hh" #include "Offline/RecoDataProducts/inc/CrvCoincidenceCluster.hh" #include "Offline/RecoDataProducts/inc/CrvDAQerror.hh" @@ -38,6 +40,7 @@ #include #include +#include #include #include @@ -172,9 +175,13 @@ namespace mu2e using Comment=fhicl::Comment; fhicl::Atom useDQMcollector{Name("useDQMcollector"), Comment("fill DQM values, histograms, ...")}; fhicl::Atom crvDigiModuleLabel{Name("crvDigiModuleLabel"), Comment("label of CrvDigi module")}; + fhicl::Atom crvStatusModuleLabel{Name("crvStatusModuleLabel"), Comment("label of CrvStatus module; empty uses crvDigiModuleLabel"), ""}; //fhicl::Atom crvRecoPulsesModuleLabel{Name("crvRecoPulsesModuleLabel"), Comment("label of CrvReco module")}; fhicl::Atom crvCoincidenceClusterFinderModuleLabel{Name("crvCoincidenceClusterFinderModuleLabel"), Comment("label of CoincidenceClusterFinder module")}; fhicl::Atom crvDaqErrorModuleLabel{Name("crvDaqErrorModuleLabel"), Comment("label of module that found the CRV-DAQ errors")}; + fhicl::Atom crvDigiDQMDir{Name("crvDigiDQMDir"), Comment("TFileService subdirectory for CRVDigiDQM histograms"), "CRVDigiDQM"}; + fhicl::Atom fillInclusiveDigiDQM{Name("fillInclusiveDigiDQM"), Comment("also fill BarId/SiPM/ADC in CRVDigiDQM"), true}; + fhicl::Atom crvDigiDQMkppReadout{Name("crvDigiDQMkppReadout"), Comment("KPP FEB-axis sizing (ROC 1-2); ROC4->ROC2 is the unpacker's job"), true}; fhicl::Atom histPEsBins{Name("histPEsBins"), Comment("number of bins for PE histograms"), 75}; fhicl::Atom histPEsStart{Name("histPEsStart"), Comment("range start for PE histograms"), 0}; @@ -196,16 +203,19 @@ namespace mu2e typedef art::EDAnalyzer::Table Parameters; explicit CrvDQMcollector(const Parameters& config); - void analyze(const art::Event& e); - void beginRun(const art::Run &run); - void endJob(); + void beginJob() override; + void analyze(const art::Event& e) override; + void beginRun(const art::Run &run) override; + void endJob() override; private: bool _useDQMcollector; std::string _crvDigiModuleLabel; + std::string _crvStatusModuleLabel; //std::string _crvRecoPulsesModuleLabel; std::string _crvCoincidenceClusterFinderModuleLabel; std::string _crvDaqErrorModuleLabel; + std::string _crvDigiDQMDir; int _histPEsBins; double _histPEsStart; @@ -230,19 +240,14 @@ namespace mu2e std::pair _lastRunSubrun; std::vector _nCoincidences; //for each sector - std::vector _nDigis; //for each channel - std::vector _nDigisROC; //for each channel std::vector _histPEs; //for each channel std::vector _histPEsROC; //for each channel std::vector _notConnected; //for each channel - std::vector _histDigisPerChannelAndEvent; - std::vector _histDigiRatesROC; std::vector _histPEsMPV; std::vector _histPEsMPVROC; std::vector _histPedestals; std::vector _histCalibConstants; - TH2F* _hist2DDigiRatesROC; TH2F* _hist2DPEsMPVROC; TH1I* _histCoincidenceClusters; TTree* _treeMetaData; @@ -250,15 +255,20 @@ namespace mu2e ProditionsHandle _calib; ProditionsHandle _sipmStatus; ProditionsHandle _channelMap_h; + + CRVDigiDQM _digiDQM; }; CrvDQMcollector::CrvDQMcollector(const Parameters& conf) : art::EDAnalyzer(conf), _useDQMcollector(conf().useDQMcollector()), _crvDigiModuleLabel(conf().crvDigiModuleLabel()), + _crvStatusModuleLabel(conf().crvStatusModuleLabel().empty() ? + conf().crvDigiModuleLabel() : conf().crvStatusModuleLabel()), //_crvRecoPulsesModuleLabel(conf().crvRecoPulsesModuleLabel()), _crvCoincidenceClusterFinderModuleLabel(conf().crvCoincidenceClusterFinderModuleLabel()), _crvDaqErrorModuleLabel(conf().crvDaqErrorModuleLabel()), + _crvDigiDQMDir(conf().crvDigiDQMDir()), _histPEsBins(conf().histPEsBins()), _histPEsStart(conf().histPEsStart()), _histPEsEnd(conf().histPEsEnd()), @@ -276,12 +286,31 @@ namespace mu2e _PEstart(conf().PEstart()), _totalEvents(0), _totalEventsWithCoincidenceClusters(0), - _totalEventsWithDAQerrors(0) + _totalEventsWithDAQerrors(0), + _hist2DPEsMPVROC(nullptr), + _histCoincidenceClusters(nullptr), + _treeMetaData(nullptr), + _digiDQM([] (bool fillInclusive, bool kppReadout) { + CRVDigiDQM::Config c; + c.fillInclusive = fillInclusive; + c.kppReadout = kppReadout; + return c; + }(conf().fillInclusiveDigiDQM(), conf().crvDigiDQMkppReadout())) { } + void CrvDQMcollector::beginJob() + { + art::ServiceHandle tfs; + _digiDQM.Book(tfs->mkdir(_crvDigiDQMDir)); + } + void CrvDQMcollector::endJob() { + _digiDQM.WriteGraphs(); + + if(_totalEvents<=0) return; + GeomHandle CRS; auto &crvCounters = CRS->getAllCRSScintillatorBars(); for(size_t channel=0; channelgetBar(barIndex).id().getShieldNumber(); - _histDigisPerChannelAndEvent.at(sectorNumber)->Fill((float)(_nDigis.at(channel))/_totalEvents); - float MPV=0; float FWHM=0; float signals=0; @@ -301,7 +328,6 @@ namespace mu2e _histPEsMPV.at(sectorNumber)->Fill(MPV); } - art::ServiceHandle tfs; for(size_t ROC=1; ROC<=CRVId::nROC; ++ROC) { for(size_t FEB=1; FEB<=CRVId::nFEBPerROC; ++FEB) @@ -309,8 +335,6 @@ namespace mu2e { size_t ROCchannel=(FEB-1)*CRVId::nChanPerFEB+FEBchannel; - _histDigiRatesROC.at(ROC-1)->Fill(ROCchannel,(float)(_nDigisROC.at((ROC-1)*CRVId::nFEBPerROC*CRVId::nChanPerFEB+ROCchannel))/_totalEvents); - float MPV=0; float FWHM=0; float signals=0; @@ -319,8 +343,6 @@ namespace mu2e _histPEsMPVROC.at(ROC-1)->Fill(ROCchannel,MPV); size_t portIndex=(ROC-1)*CRVId::nFEBPerROC+FEB-1; - - _hist2DDigiRatesROC->Fill(FEBchannel,portIndex,(float)(_nDigisROC.at((ROC-1)*CRVId::nFEBPerROC*CRVId::nChanPerFEB+ROCchannel))/_totalEvents); _hist2DPEsMPVROC->Fill(FEBchannel,portIndex,MPV); } } @@ -339,11 +361,7 @@ namespace mu2e _histPEsMPVROC.reserve(CRVId::nROC); _histPedestals.reserve(crvSectors.size()); _histCalibConstants.reserve(crvSectors.size()); - _histDigisPerChannelAndEvent.reserve(crvSectors.size()); - _histDigiRatesROC.reserve(CRVId::nROC); _nCoincidences.resize(crvSectors.size()); - _nDigis.resize(crvCounters.size()*CRVId::nChanPerBar); - _nDigisROC.resize(CRVId::nROC*CRVId::nFEBPerROC*CRVId::nChanPerFEB); _histPEs.reserve(crvCounters.size()*CRVId::nChanPerBar); _histPEsROC.reserve(CRVId::nROC*CRVId::nFEBPerROC*CRVId::nChanPerFEB); _notConnected.resize(crvCounters.size()*CRVId::nChanPerBar); @@ -370,21 +388,14 @@ namespace mu2e _histCalibConstants.emplace_back(tfs->make(Form("crvCalibConstants_CRVsector%s",crvSectors.at(i).name("").c_str()), Form("crvCalibConstants_CRVsector%s",crvSectors.at(i).name("").c_str()), _histCalibConstsBins,_histCalibConstsStart,_histCalibConstsEnd)); - _histDigisPerChannelAndEvent.emplace_back(tfs->make(Form("crvDigisPerChannelAndEvent_CRVsector%s",crvSectors.at(i).name("").c_str()), - Form("crvDigisPerChannelAndEvent_CRVsector%s",crvSectors.at(i).name("").c_str()), - _histDigisBins,_histDigisStart,_histDigisEnd)); } for(size_t ROC=1; ROC<=CRVId::nROC; ++ROC) { _histPEsMPVROC.emplace_back(tfs->make(Form("crvPEsMPV_ROC%zu",ROC), Form("crvPEsMPV_ROC%zu",ROC), CRVId::nFEBPerROC*CRVId::nChanPerFEB,0,CRVId::nFEBPerROC*CRVId::nChanPerFEB)); - _histDigiRatesROC.emplace_back(tfs->make(Form("crvDigiRates_ROC%zu",ROC), - Form("crvDigiRates_ROC%zu",ROC), - CRVId::nFEBPerROC*CRVId::nChanPerFEB,0,CRVId::nFEBPerROC*CRVId::nChanPerFEB)); } _hist2DPEsMPVROC=tfs->make("crvPEsMPV","crvPEsMPV:FEBchannel:FEB", CRVId::nChanPerFEB,0,CRVId::nChanPerFEB, CRVId::nROC*CRVId::nFEBPerROC,0,CRVId::nROC*CRVId::nFEBPerROC); - _hist2DDigiRatesROC=tfs->make("crvDigiRates","crvDigiRates:FEBchannel:FEB", CRVId::nChanPerFEB,0,CRVId::nChanPerFEB, CRVId::nROC*CRVId::nFEBPerROC,0,CRVId::nROC*CRVId::nFEBPerROC); _histCoincidenceClusters=tfs->make("crvCoincidencesClusters","crvCoincidenceClusters:sectorType",10,0,10); _treeMetaData=tfs->make("crvMetaData","crvMetaData"); @@ -409,6 +420,8 @@ namespace mu2e art::Handle crvDaqErrorCollection; event.getByLabel(_crvDigiModuleLabel,"",crvDigiCollection); + art::Handle crvStatusCollection; + event.getByLabel(_crvStatusModuleLabel,"",crvStatusCollection); //event.getByLabel(_crvRecoPulsesModuleLabel,"",crvRecoPulseCollection); event.getByLabel(_crvCoincidenceClusterFinderModuleLabel,"",crvCoincidenceClusterCollection); event.getByLabel(_crvDaqErrorModuleLabel,"",crvDaqErrorCollection); @@ -417,20 +430,15 @@ namespace mu2e auto const& sipmStatus = _sipmStatus.get(event.id()); auto const& channelMap = _channelMap_h.get(event.id()); - for(size_t i=0; isize(); ++i) - { - const CrvDigi &digi = crvDigiCollection->at(i); - int barIndex = digi.GetScintillatorBarIndex().asUint(); - int SiPM = digi.GetSiPMNumber(); - size_t channel = barIndex*CRVId::nChanPerBar + SiPM; - ++_nDigis.at(channel); - - int ROC=digi.GetROC(); - int ROCport=digi.GetFEB(); - int FEBchannel=digi.GetFEBchannel(); - size_t channelOnline = (ROC-1)*CRVId::nFEBPerROC*CRVId::nChanPerFEB + (ROCport-1)*CRVId::nChanPerFEB + FEBchannel; - ++_nDigisROC.at(channelOnline); - } + const CrvStatusCollection emptyStatus; + const CrvDigiCollection emptyDigis; + const CrvDigiCollection& digis = + (crvDigiCollection.isValid() && crvDigiCollection.product()!=nullptr) ? + *crvDigiCollection : emptyDigis; + const CrvStatusCollection& status = + (crvStatusCollection.isValid() && crvStatusCollection.product()!=nullptr) ? + *crvStatusCollection : emptyStatus; + _digiDQM.Fill(digis, status); static bool first=true; if(first) @@ -455,6 +463,22 @@ namespace mu2e _histPedestals.at(sectorNumber)->Fill(pedestal); _histCalibConstants.at(sectorNumber)->Fill(calibPulseArea); } + //helper owns crvDigisPerChannelAndEvent_*; -1 drops notConnected channels + std::vector sectorNames; + auto &crvSectors = CRS->getCRSScintillatorShields(); + sectorNames.reserve(crvSectors.size()); + for(size_t i=0; i channelToSector(crvCounters.size()*CRVId::nChanPerBar, -1); + for(size_t channel=0; channelgetBar(barIndex).id().getShieldNumber(); + } + _digiDQM.BookSectorOccupancy(sectorNames, channelToSector, + _histDigisBins, _histDigisStart, _histDigisEnd); + _firstRunSubrun=std::pair(event.run(),event.subRun()); } _lastRunSubrun=std::pair(event.run(),event.subRun()); diff --git a/CRVReco/src/SConscript b/CRVReco/src/SConscript index fbb4148bbf..ed18e4c2b0 100644 --- a/CRVReco/src/SConscript +++ b/CRVReco/src/SConscript @@ -29,6 +29,7 @@ mainlib = helper.make_mainlib ( [ 'CLHEP', ) helper.make_plugins( [ mainlib, + 'mu2e_DQMHelpers', 'mu2e_SeedService', 'mu2e_Mu2eUtilities', 'mu2e_GeometryService', diff --git a/DQMHelpers/CMakeLists.txt b/DQMHelpers/CMakeLists.txt new file mode 100644 index 0000000000..0446274cd3 --- /dev/null +++ b/DQMHelpers/CMakeLists.txt @@ -0,0 +1,32 @@ +cet_make_library( + SOURCE + src/CRVDigiDQM.cc + src/CRVStatusDQM.cc + LIBRARIES PUBLIC + Offline::RecoDataProducts + Offline::DataProducts + art_root_io::tfile_support + messagefacility::MF_MessageLogger + ROOT::Hist + ROOT::Gpad +) + +cet_build_plugin(CRVDigiDQMAnalyzer art::module + REG_SOURCE src/CRVDigiDQMAnalyzer_module.cc + LIBRARIES REG + Offline::DQMHelpers + Offline::GeometryService + Offline::CosmicRayShieldGeom + art_root_io::TFileService_service +) + +cet_build_plugin(CRVStatusDQMAnalyzer art::module + REG_SOURCE src/CRVStatusDQMAnalyzer_module.cc + LIBRARIES REG + Offline::DQMHelpers + art_root_io::TFileService_service +) + +install_source(SUBDIRS src) +install_headers(USE_PROJECT_NAME SUBDIRS inc) +install_fhicl(SUBDIRS fcl SUBDIRNAME Offline/DQMHelpers/fcl) diff --git a/DQMHelpers/README.md b/DQMHelpers/README.md new file mode 100644 index 0000000000..d4521d7f2d --- /dev/null +++ b/DQMHelpers/README.md @@ -0,0 +1,243 @@ +# DQMHelpers + +Shared DQM histogram helpers used by both offline art analyzers and the +otsdaq online monitor. The helper classes own histogram booking and filling; +art modules own I/O, visualization, and histogram shipping. + +This package has no GeometryService or ProditionsService dependency, so the +same fill path can run in the DAQ process. + +## Output file layout + +`TFileService` writes **one file per art job** and gives **each module its own +directory**, named after the module label. That directory is automatic and not +optional — it is what keeps two modules from colliding in one file. + +```text +crvDQM.root + CRVDigiDQM/ h1_peakAdc, h1_tdc, dtVsFeb, crvDigiRates*, timing_fpga/ ... + CRVStatusDQM/ nRocHeaders, errorBitsVsRoc, linkLatency* ... +``` + +`outputTag` adds a *second* level inside the module directory. Use it only when +the module books histograms of its own alongside the helper's, so the two owners +stay separated: + +| Module | `outputTag` | Why | +|---|---|---| +| `CRVStatusDQMAnalyzer` | `""` | books nothing itself — a second level would just repeat the module label | +| `CRVDigiDQMAnalyzer` | `""` | as above; set it if `fillSectorOccupancy` is on and you want the per-sector hists kept apart | +| `CrvDQMcollector` | `"CRVDigiDQM"` | **keep** — the module also books `crvPEsMPV*`, `crvPedestals*`, `crvCalibConstants*` and the `crvMetaData` tree at module level | + +### Per-sector occupancy + +`crvDigisPerChannelAndEvent_CRVsector*` is owned by the helper. Sector names and the +channel->sector map are geometry-derived, so the caller injects them via +`BookSectorOccupancy(...)` and the helper stays free of GeometryService. A +negative sector entry skips that channel, which is how `CrvDQMcollector` drops +its Proditions `notConnected` channels without the helper knowing about +Proditions. `WriteGraphs()` fills them from `nDigisOffline`. + +An empty `outputTag` books directly in the module directory. The module labels in +`CRVDQM.fcl` are therefore `CRVDigiDQM` / `CRVStatusDQM`, so the directory name is +the same either way and `crv_status_extractor.py --dir CRVStatusDQM` keeps working +(it matches a top-level key or any key ending in `/CRVStatusDQM`). + +`CRVDQM.fcl` is the only FCL in this package: digi and status always run together +in one job, into one file. Running one side alone is a matter of dropping the +other module from `physics.ana`, not a separate config to keep in step. + +## CRVDigiDQM + +`mu2e::CRVDigiDQM` books and fills every live digi histogram from +`otsdaq-mu2e-crv` `CrvDQM_module.cc` (`mu2e/ots_ops`), plus optional +`ValCrvDigi` inclusive plots (`BarId`, `SiPM`, `ADC`) gated by +`Config::fillInclusive`. + +Typical use from an art module: + +```cpp +CRVDigiDQM dqm(config); // constructor / beginJob +dqm.Book(tfs->mkdir("CRVDigiDQM")); +dqm.Fill(*digis, *status); // analyze, once per event +dqm.WriteGraphs(); // endJob (TGraph is not auto-saved) +``` + +`CRVReco/src/CrvDQMcollector_module.cc` uses this helper for all per-event +digi histograms and the CRVId rate maps (`crvDigiRates_ROC*`, 2D +`crvDigiRates`, `crvDigisPerChannel`). The collector still owns reco/PE/ +coincidence products. Per-sector `crvDigisPerChannelAndEvent_CRVsector*` is +owned by the helper; the modules only inject the geometry-derived sector names +and channel->sector map. The helper itself has no GeometryService or Proditions +dependency. + +`Config::fillCrvIdRates` (default true) books the two rate maps and the +detector-wide 1D vs offline channel. `WriteGraphs()` scales those three by +`1/nEvents`. + +### Readout geography (`kppReadout`) + +`Config::kppReadout` (default true) sizes the FEB axes. It selects detector +size only — it does **no** ROC remapping. + +`kppReadout: true` sizes for KPP, the extracted CRV and the only one built so +far: ROC 1-2, FEBs numbered `(roc-1)*25 + feb`, and `h1_channels` / +`h1_channelsLastEwt` / `h2_channels` booked over 33 FEB slots. Every existing +dataset wants this mode, which is why it is the default here and in +`prolog_v12.fcl`. + +`kppReadout: false` is the full CRV, which does not exist yet — a seam for when +it does, not a mode anything runs in today. The occupancy trio is **not booked** +rather than resized, since 394 of 432 FEBs would fall past the 2112-bin axis and +`crvDigisPerChannel` / `crvDigiRates` already cover the full detector correctly +binned. + +**No ROC fold here.** DTC link 3 is read out as ROC 4 and folded onto ROC 2 by +`CrvDigisFromArtdaqFragmentsFEBII` (`useROC4asROC2`) at unpack time, so digis +reach the helper with the correct ROC already. Re-folding would duplicate a +DAQ-level mapping and silently mask an unpacker configured without it; instead a +ROC 4 arriving here lands off-axis and raises the warning below. + +`h1_channelsLastEwt`, the rolling EWT-window occupancy, has no full-CRV +equivalent and would need rebasing onto the CRVId convention. Irrelevant while +KPP is the only CRV that exists. + +`dtVsFeb` is booked in both modes; its x-axis follows the same geography +(`nFebIdBins()`) so it stays legible rather than reserving 450 bins for FEBs +that do not exist. + +### Inter-FEB sync (`dtVsFeb`) + +FEBs drifting out of sync with each other is the error case these timing plots +exist to catch. Because a slip is a per-FEB offset, every pairwise difference is +just `dt(i,j) = d_j - d_i` — the N x N pair matrix carries only N independent +numbers. `dtVsFeb` therefore stores those N numbers directly: x is the full +`globalFebId`, y is that FEB's first constant-fraction hit time minus the median +of the *other* FEBs' first hit times in the same event. A slipped FEB is a +displaced vertical stripe; a whole ROC slipping is a block of adjacent stripes, +since `globalFebId` is ROC-ordered. + +The reference excludes the FEB being filled so that one bad FEB does not drag the +reference and smear its partners. Events with fewer than two FEBs are skipped +(no reference exists), and with exactly two FEBs both entries are `+/-` the pair +difference — attribution then comes from aggregating over events, since the bad +FEB is displaced against *every* partner while each partner is displaced only in +the events it shares with the bad one. + +### Axis-coverage diagnostics + +Both axes can hide entries, so the raw values are counted before `Fill` and +exposed as accessors — `maxFebIdSeen()`, `nFebIdOutOfAxis()`, `nDtOutOfRange()`, +`maxAbsDtSeen()`. Reading them beats reading an overflow bin, since +`nDtOutOfRange` and `maxAbsDtSeen` are measured on the true `dt` and so register +a slip of any size. Per-FEB attribution comes from the histograms below. + +**Off-axis FEB — one warning per job.** The first time a digi arrives from a FEB +outside the axis, one `mf::LogWarning` names the ROC, FEB and `globalFebId`, then +latches. The axis size comes from `Config::kppReadout`, so this is a fixed +configuration error: it cannot change between runs, and repeating it would add +nothing. The helper needs no run-boundary hook. This is the only thing it logs. + +### Per-FEB desync counters + +An off-scale `dt` is a physics observation, not a misconfiguration, so it is +counted rather than logged. Two `TH1F`s, both on the same `globalFebId` axis as +`dtVsFeb`, counting events in which that FEB had `abs(dt) > dtVsFebRange`: + +| Histogram | Period | +|---|---| +| `dtOutOfRangePerFeb` | whole job / file — the offline view | +| `dtOutOfRangePerFebLastEwt` | rolling `channelsWindowEwts` — the online view | + +A FEB whose clock has slipped shows a bar standing above its neighbours. The +rolling twin exists so a slip that starts now is not diluted by hours of earlier +good data; it shares the deque-and-`AddBinContent` mechanism used by +`h1_channelsLastEwt`, so like it it carries bin content without entry counts and +only updates on events that carry an EWT (i.e. not MC). + +A FEB contributes at most one entry per event, and only when it has a digi +passing `cfTime` (>= 3 samples, `peak - adcs[0] >= minAmplitude`, a leading-edge +crossing) *and* at least one other FEB in the event does too — a lone FEB has no +reference. So the count carries an occupancy term, which matters only while the +healthy baseline is non-zero; see the caveat under `dtVsFebRange` above. + +This replaces the former per-FEB-pair `timing_feb/dt_febXX_febYY` histograms. +Intra-FEB `timing_fpga/` histograms are unchanged. + +Event-window tags for the time-series plots come from +`CrvStatus::GetEventWindowTag()`. If the status collection is empty (typical +MC), occupancy / ADC / TDC histograms are still filled and the EWT graphs, +rolling occupancy, and MicroBunchStatus plots are skipped. + +## CRVStatusDQM + +`mu2e::CRVStatusDQM` books and fills ROC-firmware health histograms from +`CrvStatus` / `CrvDAQerror`. It does **not** fold `roc==4` onto `roc==2` +(status is per DTC link). The required online plot is TH2 `errorBitsVsRoc` +(firmware bits 24–31 vs `dtcId*6+linkId`). + +Typical use: + +```cpp +CRVStatusDQM dqm(config); +dqm.Book(tfs->mkdir("CRVStatusDQM")); +dqm.Fill(*status, *daqErrors); // analyze +dqm.EndSubRun(run, subrun); // endSubRun +dqm.WriteGraphs(); // endJob +``` + +Empty `CrvStatus` (typical MC) still counts `nEvents` and skips ROC/latency +fills. `lastEventRocs()` supplies the five artdaq LastPoint scalars +(`TriggerCount`, `EventWindowTag`, `ActiveFEBCount`, `MicroBunchStatus`, +`WordCount`) with names `CRV.DTC.ROC.*`. + +```text +mu2e -c Offline/DQMHelpers/fcl/CRVDQM.fcl -s +``` + +A missing `CrvStatus` product is tolerated (empty collection). A missing +`CrvDAQerror` product skips unpack-error histograms. + +## Offline analyzer + +`CRVDigiDQMAnalyzer` is a thin wrapper: + +```text +mu2e -c Offline/DQMHelpers/fcl/CRVDQM.fcl -s +``` + +A missing `CrvStatus` product is tolerated (empty collection). A missing +`CrvDigi` product skips the event. + +`fillSectorOccupancy: false` by default so the stock FCL needs no CRV +geometry. Set it true to book +`crvDigisPerChannelAndEvent_CRVsector*` from CosmicRayShield sector names +and fill them in `endJob`. The collector still skips `notConnected` +channels via Proditions; the analyzer does not. + +## otsdaq CrvDQM + +`otsdaq-mu2e-crv` `CrvDQM_module.cc` (`feature/CRVDigiDQM`, based on +`mu2e/ots_ops`) constructs a `mu2e::CRVDigiDQM` member with +`Config::fillInclusive = false`, calls `Book`/`Fill`/`WriteGraphs`, and +links `Offline::DQMHelpers`. The module still owns `h1_dummy`, +`HistoSender`, `THttpServer`/`TCanvas`/`CrvDQMStyle`, rate-log counters, +and the per-FEB timing summary canvases. + +The DAQ build must pick up an Offline that contains `DQMHelpers` (the +`off_dqm` checkout, or a later Offline tag). Constant-fraction timing +lives in `Offline/DQMHelpers/inc/CRVCFTime.hh`; the local +`ArtModules/CrvCFTime.hh` is unused by `CrvDQM`. + +Binning FHiCL defaults in `CRVDigiDQM::Config` match the online +`ps.get(...)` defaults so a cutover on the same file is comparable. + +## otsdaq CrvStatusMetrics + +`otsdaq-mu2e-crv` `CrvStatusMetrics_module.cc` constructs a +`mu2e::CRVStatusDQM` member, prefers `CrvStatus`/`CrvDAQerror` products, +and still publishes the five LastPoint series. If the status product is +absent it falls back to DTC-fragment decode for LastPoint only. +`HistoSender` ships `errorBitsVsRoc` (and the other status hists) when +`sendHists: true`. File-mode FCL: `fcl/RunCrvStatusDQM_vst_raw.fcl`. + diff --git a/DQMHelpers/fcl/CRVDQM.fcl b/DQMHelpers/fcl/CRVDQM.fcl new file mode 100644 index 0000000000..9c4eb6afcd --- /dev/null +++ b/DQMHelpers/fcl/CRVDQM.fcl @@ -0,0 +1,72 @@ +# CRV DQM: digi and ROC-status histograms in one output file. +# +# mu2e -c Offline/DQMHelpers/fcl/CRVDQM.fcl -s -n + +#include "Offline/fcl/minimalMessageService.fcl" +#include "Offline/fcl/standardServices.fcl" + +process_name : CRVDQM + +source : { + module_type : RootInput + maxEvents : -1 +} + +services : { + message : @local::default_message + TFileService : { + fileName : "crvDQM.root" + } +} + +physics : { + analyzers : { + # module label == output directory name + CRVDigiDQM : { + module_type : CRVDigiDQMAnalyzer + crvDigiTag : "CrvDigi" + crvStatusTag : "CrvDigi" + outputTag : "" + diagLevel : 1 + nBinsDigisPerEvt : 200 + maxDigisPerEvt : 4000 + nBinsPeakAdc : 450 + maxPeakAdc : 4500 + nBinsTdc : 400 + maxTdc : 40000 + cfFraction : 0.20 + dtBinSize : 0.5 + dtRange : 100.0 + dtVsFebBinSize : 2.0 + dtVsFebRange : 500.0 + minAmplitude : 10 + avgBlockSize : 30 + avgGraphPoints : 1000 + channelsWindowEwts : 50000 + fillInclusive : true + fillCrvIdRates : true + # KPP FEB-axis sizing (ROC 1-2); false only for a full CRV + kppReadout : true + fillSectorOccupancy : false + } + + CRVStatusDQM : { + module_type : CRVStatusDQMAnalyzer + crvStatusTag : "CrvDigi" + crvDaqErrorTag : "CrvDigi" + outputTag : "" + diagLevel : 1 + nBinsLatency : 1024 + maxLinkLatency : 4096.0 + nBinsTriggerCount : 256 + maxTriggerCount : 65535.0 + nBinsWordCount : 256 + maxWordCount : 65535.0 + nBinsEwtMismatch : 201 + maxEwtMismatch : 100.0 + } + } + + ana : [ CRVDigiDQM, CRVStatusDQM ] + end_paths : [ ana ] +} diff --git a/DQMHelpers/inc/CRVCFTime.hh b/DQMHelpers/inc/CRVCFTime.hh new file mode 100644 index 0000000000..49a9c87e88 --- /dev/null +++ b/DQMHelpers/inc/CRVCFTime.hh @@ -0,0 +1,64 @@ +#ifndef DQMHelpers_inc_CRVCFTime_hh +#define DQMHelpers_inc_CRVCFTime_hh +// Constant-fraction timing for CRV waveforms. Returns the time within the +// waveform in ns; the caller adds startTDC * digitizationPeriod. +// Ported from otsdaq-mu2e-crv ArtModules/CrvCFTime.hh (mu2e/ots_ops). + +#include +#include +#include +#include +#include + +namespace mu2e { + +constexpr double kDigitizationPeriodNs = 12.5; // ns per TDC count / ADC sample + +struct CFResult { + double time_ns{ + std::numeric_limits::quiet_NaN()}; // time within waveform [ns] + bool valid{false}; + int16_t baseline{0}; + int16_t peak{0}; +}; + +inline CFResult cfTime(const std::vector& adcs, + double fraction = 0.20, + int minAmplitude = 0, + double digitizationPeriod = kDigitizationPeriodNs) +{ + CFResult r; + if (adcs.size() < 3) { + return r; + } + + r.baseline = adcs[0]; + + auto it = std::max_element(adcs.begin(), adcs.end()); + r.peak = *it; + + double amplitude = static_cast(r.peak) - r.baseline; + if (amplitude <= 0 || amplitude < minAmplitude) { + return r; + } + + double threshold = r.baseline + fraction * amplitude; + + std::size_t peakIdx = static_cast(std::distance(adcs.begin(), it)); + + for (std::size_t i = 1; i <= peakIdx; ++i) { + if (adcs[i] >= threshold && adcs[i - 1] < threshold) { + double denom = static_cast(adcs[i]) - adcs[i - 1]; + double frac = (denom != 0.0) ? (threshold - adcs[i - 1]) / denom : 0.0; + r.time_ns = ((i - 1) + frac) * digitizationPeriod; + r.valid = true; + return r; + } + } + + return r; +} + +} // namespace mu2e + +#endif /* DQMHelpers_inc_CRVCFTime_hh */ diff --git a/DQMHelpers/inc/CRVDigiDQM.hh b/DQMHelpers/inc/CRVDigiDQM.hh new file mode 100644 index 0000000000..dbea60f6d0 --- /dev/null +++ b/DQMHelpers/inc/CRVDigiDQM.hh @@ -0,0 +1,237 @@ +#ifndef DQMHelpers_inc_CRVDigiDQM_hh +#define DQMHelpers_inc_CRVDigiDQM_hh +// Standalone CRV digi DQM helper: books and fills the histograms used by the +// otsdaq online monitor and the offline DQM modules. No GeometryService. +// Original Author: R. Mina + +#include "Offline/DataProducts/inc/CRVId.hh" +#include "Offline/RecoDataProducts/inc/CrvDigi.hh" +#include "Offline/RecoDataProducts/inc/CrvStatus.hh" + +#include "art_root_io/TFileDirectory.h" + +#include "TH1D.h" +#include "TH1F.h" +#include "TH2F.h" +#include "TGraph.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mu2e { + +class CRVDigiDQM { +public: + // Binning and CF-timing defaults match otsdaq-mu2e-crv CrvDQM_module.cc + // (branch mu2e/ots_ops). + struct Config { + int nBinsDigisPerEvt{200}; + float maxDigisPerEvt{4000.f}; + int nBinsPeakAdc{450}; + float maxPeakAdc{4500.f}; + int nBinsTdc{400}; + float maxTdc{40000.f}; + double cfFraction{0.20}; + float dtBinSize{0.5f}; + float dtRange{100.f}; + // dtVsFeb needs a range wide enough to show a firmware clock slip, which is + // much larger than the few-ns spread dtRange is binned for. + float dtVsFebBinSize{2.f}; + float dtVsFebRange{500.f}; + int minAmplitude{10}; + std::size_t avgBlockSize{30}; + std::size_t avgGraphPoints{1000}; + std::size_t channelsWindowEwts{50000}; + bool fillInclusive{true}; + // CRVId-indexed rate maps and offline-channel occupancy (no GeometryService). + bool fillCrvIdRates{true}; + // KPP FEB-axis sizing: book the 33-slot occupancy pair and size the FEB + // axes for ROC 1-2. False for full CRV; see the header comment above. + bool kppReadout{true}; + }; + + // KPP readout geography used by the online occupancy plots. + static constexpr int kNFebSlotsPerROC = 25; + static constexpr int kNChanPerFEB = 64; + static constexpr int kNGlobalChannelBins = 2112; + static constexpr int kNGlobalFebBins = 32; + + // Full-CRV globalFebId range, used only when kppReadout is false. + static constexpr int kNFebIdBins = kNFebSlotsPerROC * static_cast(CRVId::nROC); + + // dtVsFeb x-axis: same geography as the occupancy axes, so both modes stay + // readable and the two stay consistent by construction. + int nFebIdBins() const + { + return config_.kppReadout ? kNGlobalFebBins + 1 : kNFebIdBins; + } + + static constexpr std::size_t kEwtWindow = 1000; + static constexpr std::size_t kGraphPoints = 10000; + static constexpr double kEwtXRange = 1000000; + + static int globalFebId(uint8_t roc, uint8_t feb); + static int globalChannelId(uint8_t roc, uint8_t feb, uint8_t febChannel); + + explicit CRVDigiDQM(const Config& config); + + void Book(art::TFileDirectory dir); + // Caller injects the geometry-derived sector map so the helper needs no + // GeometryService. Negative sector skips the channel. Call once after Book(). + void BookSectorOccupancy(const std::vector& sectorNames, + const std::vector& channelToSector, + int nBins, double lo, double hi); + //digis/event per channel, one hist per CRV sector + const std::vector& sectorOccupancy() const { return h_sectorOccupancy_; } + void Fill(const CrvDigiCollection& crvDigis, + const CrvStatusCollection& crvStatus); + void WriteGraphs(); + + TH1F* h1_digisPerEvt() const { return h1_digisPerEvt_; } //digis per event + TH1F* h1_peakAdc() const { return h1_peakAdc_; } //largest ADC sample of a digi + TH1F* h1_tdc() const { return h1_tdc_; } //digi start time in 12.5ns ticks + TH1F* h1_channels() const { return h1_channels_; } //occupancy vs global channel ID + TH1F* h1_channelsLastEwt() const { return h1_channelsLastEwt_; } //same, rolling EWT window + TH2F* h2_channels() const { return h2_channels_; } //FEB vs FEB channel hit map + TGraph* g_digisVsEwt() const { return g_digisVsEwt_; } //digis in the last EWTs vs EWT + TGraph* g_digisAvgVsEwt() const { return g_digisAvgVsEwt_; } //mean digis/event vs EWT + + TH1D* BarId() const { return hBarId_; } //ValCrvDigi: scintillator bar index + TH1D* SiPM() const { return hSiPM_; } //ValCrvDigi: SiPM number within the bar + TH1D* ADC() const { return hADC_; } //ValCrvDigi: every ADC sample + + //CRVId rate maps, scaled to digis/event in WriteGraphs + TH1F* crvDigisPerChannel() const { return h_crvDigisPerChannel_; } //vs offline channel bar*4+SiPM + TH2F* crvDigiRates() const { return h_crvDigiRates_; } //FEB channel vs FEB port + const std::vector& crvDigiRatesROC() const { return h_crvDigiRatesROC_; } //one per ROC + int nDigisOffline(std::size_t channel) const; + std::size_t nOfflineChannels() const { return nDigisOffline_.size(); } + bool ratesScaled() const { return ratesScaled_; } + + //inter-FEB sync: first-hit time minus the median of the other FEBs, per FEB + TH2F* dtVsFeb() const { return h2_dtVsFeb_; } + + //a FEB whose clock has slipped stands above the others in these + TH1F* dtOutOfRangePerFeb() const { return h_dtOutOfRangePerFeb_; } //events with |dt| off scale + TH1F* dtOutOfRangePerFebLastEwt() const //same, rolling EWT window + { + return h_dtOutOfRangePerFebLastEwt_; + } + //intra-FEB timing, keyed (globalFebId, fpgaA*4+fpgaB) + const std::map, TH1F*>& dtFpgaPairs() const + { + return h1_dtFpgaPairs_; + } + //ROC MicroBunchStatus vs EWT, one graph per DTC link + const std::map& ubStatusVsEwt() const + { + return g_ubStatusVsEwt_; + } + + // Axis-coverage diagnostics. Only the off-axis FEB case is logged (once per + // job, from Fill); the dt counts are reported through the histograms above. + int maxFebIdSeen() const { return maxFebIdSeen_; } + long long nFebIdOutOfAxis() const { return nFebIdOutOfAxis_; } + long long nDtOutOfRange() const { return nDtOutOfRange_; } + double maxAbsDtSeen() const { return maxAbsDtSeen_; } + + std::size_t nEvents() const { return nEvents_; } + std::size_t nDigis() const { return nDigis_; } + bool hasEwtWindow() const { return !ewtWindow_.empty(); } + uint64_t lastEwt() const + { + return ewtWindow_.empty() ? 0 : ewtWindow_.back().first; + } + const std::set& activeFEBs() const { return activeFEBs_; } + const std::set& activeROCs() const { return activeROCs_; } + const std::map>& rocFEBMap() const + { + return rocFEBMap_; + } + +private: + struct FpgaHit { + double time_ns; + uint8_t channel; + }; + + void fillEwtSeries(uint64_t ewt, int nDigis); + void fillRollingOccupancy(uint64_t ewt, + const std::vector& eventChannelHits); + void fillTiming(const std::map>>& hitTimes); + void fillMicroBunchStatus(const CrvStatusCollection& crvStatus); + void persistGraph(TGraph* g); + void scaleRateHists(); + void fillRollingDtOutOfRange(uint64_t ewt); + void fillSectorOccupancy(); + + Config config_; + int nBinsDt_{400}; + int nBinsDtVsFeb_{500}; + bool booked_{false}; + + std::optional dir_; + std::optional timingFpgaDir_; + + TH1F* h1_digisPerEvt_{nullptr}; + TH1F* h1_peakAdc_{nullptr}; + TH1F* h1_tdc_{nullptr}; + TH1F* h1_channels_{nullptr}; + TH1F* h1_channelsLastEwt_{nullptr}; + TH2F* h2_channels_{nullptr}; + TGraph* g_digisVsEwt_{nullptr}; + TGraph* g_digisAvgVsEwt_{nullptr}; + + TH1D* hBarId_{nullptr}; + TH1D* hSiPM_{nullptr}; + TH1D* hADC_{nullptr}; + + TH1F* h_crvDigisPerChannel_{nullptr}; + TH2F* h_crvDigiRates_{nullptr}; + std::vector h_crvDigiRatesROC_; + std::vector nDigisOffline_; + bool ratesScaled_{false}; + + TH2F* h2_dtVsFeb_{nullptr}; + std::vector h_sectorOccupancy_; + std::vector channelToSector_; + TH1F* h_dtOutOfRangePerFeb_{nullptr}; + TH1F* h_dtOutOfRangePerFebLastEwt_{nullptr}; + std::map, TH1F*> h1_dtFpgaPairs_; + std::map g_ubStatusVsEwt_; + std::map lastMicroBunchStatus_; + + bool warnedOffAxisFeb_{false}; + int maxFebIdSeen_{-1}; + long long nFebIdOutOfAxis_{0}; + long long nDtOutOfRange_{0}; + double maxAbsDtSeen_{0.0}; + // FEBs whose dt was out of range in the event being filled. + std::vector dtOutOfRangeThisEvent_; + + std::size_t nEvents_{0}; + std::size_t nDigis_{0}; + std::set activeFEBs_; + std::set activeROCs_; + std::map> rocFEBMap_; + + std::deque> ewtWindow_; + long long ewtWindowSum_{0}; + std::deque>> recentChannelHitsByEwt_; + std::deque>> recentDtByEwt_; + + long long avgBlockSum_{0}; + std::size_t avgBlockCount_{0}; + uint64_t avgBlockFirstEwt_{0}; + bool avgSeedsCleared_{false}; +}; + +} // namespace mu2e + +#endif /* DQMHelpers_inc_CRVDigiDQM_hh */ diff --git a/DQMHelpers/inc/CRVStatusDQM.hh b/DQMHelpers/inc/CRVStatusDQM.hh new file mode 100644 index 0000000000..f1e8e96cf3 --- /dev/null +++ b/DQMHelpers/inc/CRVStatusDQM.hh @@ -0,0 +1,162 @@ +#ifndef DQMHelpers_inc_CRVStatusDQM_hh +#define DQMHelpers_inc_CRVStatusDQM_hh +// Standalone CRV ROC-status DQM helper: firmware-health histograms for the +// otsdaq online monitor and CRVStatusDQMAnalyzer. Status is per DTC link. +// Original Author: R. Mina + +#include "Offline/RecoDataProducts/inc/CrvDAQerror.hh" +#include "Offline/RecoDataProducts/inc/CrvStatus.hh" + +#include "art_root_io/TFileDirectory.h" + +#include "TH1F.h" +#include "TH2F.h" +#include "TGraph.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mu2e { + +class CRVStatusDQM { +public: + struct Config { + int nBinsLatency{1024}; + float maxLinkLatency{4096.f}; + int nBinsTriggerCount{256}; + float maxTriggerCount{65535.f}; + int nBinsWordCount{256}; + float maxWordCount{65535.f}; + int nBinsEwtMismatch{201}; + float maxEwtMismatch{100.f}; + }; + + static constexpr int kNErrorBits = 8; + static constexpr int kErrorBitOffset = 24; + static constexpr int kNPortFlags = 24; + static constexpr int kNLinksPerDTC = 6; + static constexpr int kNRocBins = 18; // CRVId::nROC; no GeometryService + static constexpr int kNDaqErrorCodes = 7; + + static const char* errorBitLabel(int bitIndex); + // Y-axis of errorBitsVsRoc: dtcId * 6 + linkId. No roc==4 fold. + static int rocBin(uint8_t dtcId, uint8_t linkId); + + struct RocSnapshot { + uint8_t dtcId{0}; + uint8_t linkId{0}; + uint64_t ewt{0}; + uint16_t triggerCount{0}; + uint16_t wordCount{0}; + uint16_t activeFebCount{0}; + uint16_t linkLatency{0}; + uint32_t microBunchStatus{0}; + }; + + explicit CRVStatusDQM(const Config& config); + + void Book(art::TFileDirectory dir); + void Fill(const CrvStatusCollection& crvStatus); + void Fill(const CrvStatusCollection& crvStatus, + const CrvDAQerrorCollection& crvDaqErrors); + void EndSubRun(int run, int subrun); + void WriteGraphs(); + + TH1F* nRocHeaders() const { return h_nRocHeaders_; } //ROC headers per event + TH1F* activeFebCount() const { return h_activeFebCount_; } //active FEBs per ROC header + TH1F* triggerCount() const { return h_triggerCount_; } //ROC TriggerCount word + TH1F* wordCount() const { return h_wordCount_; } //ROC ControllerEventWordCount word + TH1F* linkLatency() const { return h_linkLatency_; } //DTC link latency, all links + TH1F* errorBits() const { return h_errorBits_; } //firmware error bits 24-31, labelled + TH2F* errorBitsVsRoc() const { return h_errorBitsVsRoc_; } //those bits vs dtcId*6+linkId + TH1F* portFlags() const { return h_portFlags_; } //per-port problem flags, bits 0-23 + TH1F* rocCensus() const { return h_rocCensus_; } //ROC headers seen per dtcId*6+linkId + TH1F* eventHasError() const { return h_eventHasError_; } //0/1 per event: any firmware bit + TH1F* eventHasDaqError() const { return h_eventHasDaqError_; } //0/1 per event: unpack error + TH1F* daqErrorCode() const { return h_daqErrorCode_; } //CrvDAQerror code, labelled + TH1F* ewtMismatch() const { return h_ewtMismatch_; } //ROC EWT minus DTC EWT + TH1F* errorsPerSubrun() const { return h_errorsPerSubrun_; } //error events in a subrun + TH1F* meanLatencyPerSubrun() const { return h_meanLatencyPerSubrun_; } //mean latency in a subrun + TGraph* errorsVsSubrun() const { return g_errorsVsSubrun_; } //error events vs subrun + TGraph* meanLatencyVsSubrun() const { return g_meanLatencyVsSubrun_; } //mean latency vs subrun + + //link latency, one hist per (dtcId, linkId) + const std::map, TH1F*>& linkLatencyByRoc() const + { + return h_linkLatencyByRoc_; + } + + const std::vector& lastEventRocs() const { return lastEventRocs_; } + + std::size_t nEvents() const { return nEvents_; } + std::size_t nEventsWithRocHeader() const { return nEventsWithRocHeader_; } + std::size_t nEventsWithAnyErrorBit() const { return nEventsWithAnyErrorBit_; } + std::size_t nEventsWithDaqErrors() const { return nEventsWithDaqErrors_; } + std::size_t nRocHeadersTotal() const { return nRocHeadersTotal_; } + uint16_t nActiveFEBsMin() const { return nActiveFEBsMin_; } + uint16_t nActiveFEBsMax() const { return nActiveFEBsMax_; } + double nActiveFEBsMean() const; + const std::set>& seenRocs() const + { + return seenRocs_; + } + std::size_t errorBitCount(int bitIndex) const; + bool booked() const { return booked_; } + +private: + void fillDaqErrors(const CrvDAQerrorCollection& crvDaqErrors); + void persistGraph(TGraph* g); + TH1F* latencyHistFor(uint8_t dtcId, uint8_t linkId); + + Config config_; + bool booked_{false}; + std::optional dir_; + + TH1F* h_nRocHeaders_{nullptr}; + TH1F* h_activeFebCount_{nullptr}; + TH1F* h_triggerCount_{nullptr}; + TH1F* h_wordCount_{nullptr}; + TH1F* h_linkLatency_{nullptr}; + TH1F* h_errorBits_{nullptr}; + TH2F* h_errorBitsVsRoc_{nullptr}; + TH1F* h_portFlags_{nullptr}; + TH1F* h_rocCensus_{nullptr}; + TH1F* h_eventHasError_{nullptr}; + TH1F* h_eventHasDaqError_{nullptr}; + TH1F* h_daqErrorCode_{nullptr}; + TH1F* h_ewtMismatch_{nullptr}; + TH1F* h_errorsPerSubrun_{nullptr}; + TH1F* h_meanLatencyPerSubrun_{nullptr}; + TGraph* g_errorsVsSubrun_{nullptr}; + TGraph* g_meanLatencyVsSubrun_{nullptr}; + std::map, TH1F*> h_linkLatencyByRoc_; + + std::vector lastEventRocs_; + + std::size_t nEvents_{0}; + std::size_t nEventsWithRocHeader_{0}; + std::size_t nEventsWithAnyErrorBit_{0}; + std::size_t nEventsWithDaqErrors_{0}; + std::size_t nRocHeadersTotal_{0}; + std::size_t errorBitCounts_[kNErrorBits]{}; + uint16_t nActiveFEBsMin_{std::numeric_limits::max()}; + uint16_t nActiveFEBsMax_{0}; + std::uint64_t nActiveFEBsSum_{0}; + std::size_t nActiveFEBsSamples_{0}; + std::set> seenRocs_; + + std::size_t nEventsThisSubrun_{0}; + std::size_t nEventsWithAnyErrorBitThisSubrun_{0}; + std::uint64_t latencySumThisSubrun_{0}; + std::size_t latencyNThisSubrun_{0}; +}; + +} // namespace mu2e + +#endif /* DQMHelpers_inc_CRVStatusDQM_hh */ diff --git a/DQMHelpers/src/CRVDigiDQM.cc b/DQMHelpers/src/CRVDigiDQM.cc new file mode 100644 index 0000000000..255380b704 --- /dev/null +++ b/DQMHelpers/src/CRVDigiDQM.cc @@ -0,0 +1,688 @@ +// Standalone CRV digi DQM helper. +// +// Original Author: R. Mina + +#include "Offline/DQMHelpers/inc/CRVDigiDQM.hh" +#include "Offline/DQMHelpers/inc/CRVCFTime.hh" + +#include "messagefacility/MessageLogger/MessageLogger.h" + +#include +#include +#include +#include + +#include "TString.h" +#include "TH1.h" + +namespace mu2e { + +int CRVDigiDQM::globalFebId(uint8_t roc, uint8_t feb) +{ + return (static_cast(roc) - 1) * kNFebSlotsPerROC + feb; +} + +int CRVDigiDQM::globalChannelId(uint8_t roc, uint8_t feb, uint8_t febChannel) +{ + return globalFebId(roc, feb) * kNChanPerFEB + febChannel; +} + +CRVDigiDQM::CRVDigiDQM(const Config& config) : + config_(config), nDigisOffline_(CRVId::nChannels, 0) +{ + if (config_.channelsWindowEwts == 0) { + config_.channelsWindowEwts = 1; + } + if (config_.avgBlockSize == 0) { + config_.avgBlockSize = 1; + } + if (config_.dtBinSize > 0) { + nBinsDt_ = static_cast(2.0 * config_.dtRange / config_.dtBinSize); + } + if (nBinsDt_ < 1) { + nBinsDt_ = 1; + } + if (config_.dtVsFebBinSize > 0) { + nBinsDtVsFeb_ = + static_cast(2.0 * config_.dtVsFebRange / config_.dtVsFebBinSize); + } + if (nBinsDtVsFeb_ < 1) { + nBinsDtVsFeb_ = 1; + } +} + +void CRVDigiDQM::Book(art::TFileDirectory dir) +{ + dir_ = dir; + timingFpgaDir_ = dir.mkdir("timing_fpga"); + + h2_dtVsFeb_ = dir.make( + "dtVsFeb", + "First-hit time vs median of other FEBs;Global FEB ID;#Deltat [ns]", + nFebIdBins(), + -0.5, + nFebIdBins() - 0.5, + nBinsDtVsFeb_, + -config_.dtVsFebRange, + config_.dtVsFebRange); + + // Per-FEB desync counters: a slipped FEB stands above its neighbours. + h_dtOutOfRangePerFeb_ = dir.make( + "dtOutOfRangePerFeb", + Form("Events with |#Deltat| > %.0f ns;Global FEB ID;Events", + config_.dtVsFebRange), + nFebIdBins(), -0.5, nFebIdBins() - 0.5); + + h_dtOutOfRangePerFebLastEwt_ = dir.make( + "dtOutOfRangePerFebLastEwt", + Form("Events with |#Deltat| > %.0f ns (EWT span %zu);Global FEB ID;Events", + config_.dtVsFebRange, config_.channelsWindowEwts), + nFebIdBins(), -0.5, nFebIdBins() - 0.5); + + h1_digisPerEvt_ = dir.make("h1_digisPerEvt", + "Hits / event;Hits / event;Events", + config_.nBinsDigisPerEvt, + 0.5, + config_.maxDigisPerEvt + 0.5); + h1_digisPerEvt_->SetMinimum(0.5); + + h1_peakAdc_ = dir.make("h1_peakAdc", + "Max sample ADC;Max sample ADC;Hits", + config_.nBinsPeakAdc, + 0, + config_.maxPeakAdc); + + h1_tdc_ = dir.make( + "h1_tdc", + "Start timestamp of digi in units of 12.5ns;Start timestamp of digi;Digis", + config_.nBinsTdc, + 0, + config_.maxTdc); + + // KPP-only: these axes cover 33 FEB slots, so on full CRV they would be all + // overflow. crvDigisPerChannel / crvDigiRates cover the full detector instead. + if (config_.kppReadout) { + h1_channels_ = dir.make("h1_channels", + "Channel occupancy;Global channel ID;Hits", + kNGlobalChannelBins, + -0.5, + kNGlobalChannelBins - 0.5); + h1_channels_->SetMinimum(0.5); + + h1_channelsLastEwt_ = dir.make( + "h1_channelsLastEwt", + Form("Channel occupancy (EWT span %zu);Global channel ID;Hits", + config_.channelsWindowEwts), + kNGlobalChannelBins, + -0.5, + kNGlobalChannelBins - 0.5); + h1_channelsLastEwt_->SetMinimum(0.5); + + h2_channels_ = dir.make("h2_channels", + "FEB vs channel hit map;Channel;FEB", + kNChanPerFEB, + 0.5, + kNChanPerFEB + 0.5, + kNGlobalFebBins, + 0.5, + kNGlobalFebBins + 0.5); + } + + g_digisVsEwt_ = dir.make(); + g_digisVsEwt_->SetName("g_digisVsEwt"); + g_digisVsEwt_->SetTitle(Form("Hits in last %zu EWTs (%zu points);" + "Event window tag;Hits", + kEwtWindow, + kGraphPoints)); + g_digisVsEwt_->SetPoint(0, 0, 0); + g_digisVsEwt_->SetPoint(1, 1, 1); + + g_digisAvgVsEwt_ = dir.make(); + g_digisAvgVsEwt_->SetName("g_digisAvgVsEwt"); + g_digisAvgVsEwt_->SetTitle(Form("Mean hits per event (averaged over %zu events);" + "Event window tag; ", + config_.avgBlockSize)); + g_digisAvgVsEwt_->SetPoint(0, 0, 0); + g_digisAvgVsEwt_->SetPoint(1, 1, 1); + + if (config_.fillInclusive) { + hBarId_ = dir.make("BarId", "Bar ID", 200, -0.5, 5503.5); + hSiPM_ = dir.make("SiPM", "SiPM", 4, -0.5, 3.5); + hADC_ = dir.make("ADC", "ADC in waveform", 100, 0.0, 3000.0); + } + + if (config_.fillCrvIdRates) { + h_crvDigiRatesROC_.assign(CRVId::nROC, nullptr); + for (std::size_t roc = 1; roc <= CRVId::nROC; ++roc) { + h_crvDigiRatesROC_[roc - 1] = dir.make( + Form("crvDigiRates_ROC%zu", roc), + Form("crvDigiRates_ROC%zu;Online channel in ROC;Digis / event", roc), + static_cast(CRVId::nFEBPerROC * CRVId::nChanPerFEB), + 0, + static_cast(CRVId::nFEBPerROC * CRVId::nChanPerFEB)); + } + h_crvDigiRates_ = dir.make( + "crvDigiRates", + "crvDigiRates:FEBchannel:FEB;FEB channel;FEB port", + static_cast(CRVId::nChanPerFEB), + 0, + static_cast(CRVId::nChanPerFEB), + static_cast(CRVId::nROC * CRVId::nFEBPerROC), + 0, + static_cast(CRVId::nROC * CRVId::nFEBPerROC)); + h_crvDigisPerChannel_ = dir.make( + "crvDigisPerChannel", + "Mean digis per event vs offline channel;Offline channel (bar#times4+SiPM);Digis / event", + static_cast(CRVId::nChannels), + -0.5, + static_cast(CRVId::nChannels) - 0.5); + } + + booked_ = true; +} + +void CRVDigiDQM::Fill(const CrvDigiCollection& crvDigis, + const CrvStatusCollection& crvStatus) +{ + if (!booked_) { + return; + } + + ++nEvents_; + + const bool haveEwt = !crvStatus.empty(); + const uint64_t ewt = haveEwt ? crvStatus.front().GetEventWindowTag() : 0; + + const int nDigis = static_cast(crvDigis.size()); + std::vector eventChannelHits; + std::map>> hitTimes; + + for (const auto& digi : crvDigis) { + const uint8_t roc = digi.GetROC(); + const uint8_t feb = digi.GetFEB(); + const uint8_t febChannel = digi.GetFEBchannel(); + + const int febId = globalFebId(roc, feb); + if (febId > maxFebIdSeen_) { + maxFebIdSeen_ = febId; + } + if (febId < 0 || febId >= nFebIdBins()) { + ++nFebIdOutOfAxis_; + // Once per job: the axis size comes from Config::kppReadout, so this is + // a fixed configuration error, not a per-event condition. + if (!warnedOffAxisFeb_) { + warnedOffAxisFeb_ = true; + mf::LogWarning("CRVDigiDQM") + << "digi from ROC " << static_cast(roc) + << " FEB " << static_cast(feb) << " gives globalFebId " << febId + << ", outside the " << nFebIdBins() << "-bin FEB axis. It and any " + << "others like it are hidden in the overflow bin of dtVsFeb" + << (config_.kppReadout ? " and h2_channels" : "") << ". " + << (config_.kppReadout + ? "KPP is ROC 1-2 -- check CrvDigisFromArtdaqFragmentsFEBII " + "useROC4asROC2, or set kppReadout=false for a larger CRV." + : "Raise kNFebIdBins.") + << " Reported once per job."; + } + } + + if (config_.kppReadout) { + const int channelId = globalChannelId(roc, feb, febChannel); + h1_channels_->Fill(channelId); + eventChannelHits.push_back(static_cast(channelId)); + h2_channels_->Fill(febChannel + 1, febId); + } + + const auto& adcs = digi.GetADCs(); + if (!adcs.empty()) { + const int16_t maxSample = *std::max_element(adcs.begin(), adcs.end()); + h1_peakAdc_->Fill(maxSample); + } + + h1_tdc_->Fill(digi.GetStartTDC()); + + const int barIndex = digi.GetScintillatorBarIndex().asUint(); + const int sipm = digi.GetSiPMNumber(); + if (sipm >= 0) { + const std::size_t offlineChannel = + static_cast(barIndex) * CRVId::nChanPerBar + + static_cast(sipm); + if (offlineChannel < nDigisOffline_.size()) { + ++nDigisOffline_[offlineChannel]; + if (h_crvDigisPerChannel_) { + h_crvDigisPerChannel_->Fill(static_cast(offlineChannel)); + } + } + } + + if (config_.fillCrvIdRates) { + const int rocId = static_cast(roc); + const int febIdRaw = static_cast(feb); + const int febCh = static_cast(febChannel); + if (rocId >= 1 && static_cast(rocId) <= CRVId::nROC && + febIdRaw >= 1 && static_cast(febIdRaw) <= CRVId::nFEBPerROC && + febCh >= 0 && static_cast(febCh) < CRVId::nChanPerFEB) { + const int rocChannel = + (febIdRaw - 1) * static_cast(CRVId::nChanPerFEB) + febCh; + if (!h_crvDigiRatesROC_.empty()) { + h_crvDigiRatesROC_[static_cast(rocId) - 1]->Fill(rocChannel); + } + const int portIndex = + (rocId - 1) * static_cast(CRVId::nFEBPerROC) + febIdRaw - 1; + if (h_crvDigiRates_) { + h_crvDigiRates_->Fill(febCh, portIndex); + } + } + } + + if (config_.fillInclusive) { + if (hBarId_) { + hBarId_->Fill(digi.GetScintillatorBarIndex().asInt()); + } + if (hSiPM_) { + hSiPM_->Fill(digi.GetSiPMNumber()); + } + if (hADC_) { + for (auto a : adcs) { + hADC_->Fill(a); + } + } + } + + const CFResult cf = cfTime(adcs, config_.cfFraction, config_.minAmplitude); + if (cf.valid) { + const double absTime_ns = + cf.time_ns + digi.GetStartTDC() * kDigitizationPeriodNs; + const uint8_t fpga = febChannel / 16; + hitTimes[febId][fpga].push_back({absTime_ns, febChannel}); + } + + activeROCs_.insert(roc); + activeFEBs_.insert(febId); + rocFEBMap_[roc].insert(feb); + } + + nDigis_ += static_cast(nDigis); + h1_digisPerEvt_->Fill(nDigis); + + fillTiming(hitTimes); + + if (haveEwt) { + fillEwtSeries(ewt, nDigis); + if (config_.kppReadout) { + fillRollingOccupancy(ewt, eventChannelHits); + } + fillRollingDtOutOfRange(ewt); + fillMicroBunchStatus(crvStatus); + } +} + +void CRVDigiDQM::fillEwtSeries(uint64_t ewt, int nDigis) +{ + if (avgBlockCount_ == 0) { + avgBlockFirstEwt_ = ewt; + } + avgBlockSum_ += nDigis; + ++avgBlockCount_; + if (avgBlockCount_ >= config_.avgBlockSize) { + const double midEwt = + 0.5 * (static_cast(avgBlockFirstEwt_) + static_cast(ewt)); + const double mean = static_cast(avgBlockSum_) / + static_cast(avgBlockCount_); + + if (!avgSeedsCleared_) { + g_digisAvgVsEwt_->Set(0); + avgSeedsCleared_ = true; + } + g_digisAvgVsEwt_->SetPoint(g_digisAvgVsEwt_->GetN(), midEwt, mean); + + while (static_cast(g_digisAvgVsEwt_->GetN()) > + config_.avgGraphPoints) { + g_digisAvgVsEwt_->RemovePoint(0); + } + + avgBlockSum_ = 0; + avgBlockCount_ = 0; + } + + ewtWindow_.emplace_back(ewt, nDigis); + ewtWindowSum_ += nDigis; + while (ewtWindow_.size() > kEwtWindow) { + ewtWindowSum_ -= ewtWindow_.front().second; + ewtWindow_.pop_front(); + } + if (ewtWindow_.size() == 1 && g_digisVsEwt_->GetN() == 2) { + g_digisVsEwt_->Set(0); + } + g_digisVsEwt_->SetPoint(g_digisVsEwt_->GetN(), + static_cast(ewt), + static_cast(ewtWindowSum_)); + + while (static_cast(g_digisVsEwt_->GetN()) > kGraphPoints) { + g_digisVsEwt_->RemovePoint(0); + } + + const double currentEwt = static_cast(ewt); + double xLo = std::max(0.0, currentEwt - kEwtXRange); + double xHi = currentEwt; + if (xHi <= xLo) { + xHi = xLo + 1.0; + } + if (TH1* frame = g_digisVsEwt_->GetHistogram()) { + frame->GetXaxis()->SetLimits(xLo, xHi); + } + + if (g_digisAvgVsEwt_->GetN() > 0) { + double* ax = g_digisAvgVsEwt_->GetX(); + const int nAvg = g_digisAvgVsEwt_->GetN(); + double aLo = *std::min_element(ax, ax + nAvg); + double aHi = *std::max_element(ax, ax + nAvg); + if (aHi <= aLo) { + aHi = aLo + 1.0; + } + if (TH1* frame = g_digisAvgVsEwt_->GetHistogram()) { + frame->GetXaxis()->SetLimits(aLo, aHi); + } + } +} + +void CRVDigiDQM::fillRollingOccupancy( + uint64_t ewt, const std::vector& eventChannelHits) +{ + if (h1_channelsLastEwt_ == nullptr) { + return; + } + + recentChannelHitsByEwt_.emplace_back(ewt, eventChannelHits); + for (const auto channelId : recentChannelHitsByEwt_.back().second) { + if (channelId < kNGlobalChannelBins) { + h1_channelsLastEwt_->AddBinContent(static_cast(channelId) + 1, 1.0); + } + } + + const uint64_t minKeepEwt = + (ewt > config_.channelsWindowEwts) ? ewt - config_.channelsWindowEwts : 0; + while (!recentChannelHitsByEwt_.empty() && + recentChannelHitsByEwt_.front().first < minKeepEwt) { + for (const auto channelId : recentChannelHitsByEwt_.front().second) { + if (channelId < kNGlobalChannelBins) { + h1_channelsLastEwt_->AddBinContent(static_cast(channelId) + 1, + -1.0); + } + } + recentChannelHitsByEwt_.pop_front(); + } +} + +// Median of sorted values with the entry at sorted position p removed. +namespace { +double medianExcluding(const std::vector& sorted, std::size_t p) +{ + const std::size_t m = sorted.size() - 1; + auto at = [&](std::size_t i) { return sorted[i < p ? i : i + 1]; }; + if (m % 2 == 1) { + return at(m / 2); + } + return 0.5 * (at(m / 2 - 1) + at(m / 2)); +} +} // namespace + +// Rolling EWT window for the online monitor, so a recent slip is not diluted. +void CRVDigiDQM::fillRollingDtOutOfRange(uint64_t ewt) +{ + if (h_dtOutOfRangePerFebLastEwt_ == nullptr) { + return; + } + + recentDtByEwt_.emplace_back(ewt, dtOutOfRangeThisEvent_); + for (const int febId : recentDtByEwt_.back().second) { + h_dtOutOfRangePerFebLastEwt_->AddBinContent(febId + 1, 1.0); + } + + const uint64_t minKeepEwt = + (ewt > config_.channelsWindowEwts) ? ewt - config_.channelsWindowEwts : 0; + while (!recentDtByEwt_.empty() && recentDtByEwt_.front().first < minKeepEwt) { + for (const int febId : recentDtByEwt_.front().second) { + h_dtOutOfRangePerFebLastEwt_->AddBinContent(febId + 1, -1.0); + } + recentDtByEwt_.pop_front(); + } +} + +void CRVDigiDQM::fillTiming( + const std::map>>& hitTimes) +{ + dtOutOfRangeThisEvent_.clear(); + + std::vector> febFirstHit; + for (const auto& [febId, fpgaMap] : hitTimes) { + // TODO: the median hit time may be a more noise-robust FEB reference than + // the first hit, which an early dark-noise pulse can hijack. + double earliest = std::numeric_limits::max(); + for (const auto& [fpga, hits] : fpgaMap) { + for (const auto& hit : hits) { + if (hit.time_ns < earliest) { + earliest = hit.time_ns; + } + } + } + febFirstHit.push_back({febId, earliest}); + } + + // Leave-one-out reference isolates the slipped FEB instead of smearing its + // partners, which a common reference would do at low FEB multiplicity. + if (h2_dtVsFeb_ != nullptr && febFirstHit.size() >= 2) { + std::vector sorted; + sorted.reserve(febFirstHit.size()); + for (const auto& entry : febFirstHit) { + sorted.push_back(entry.second); + } + std::sort(sorted.begin(), sorted.end()); + + for (const auto& entry : febFirstHit) { + const std::size_t p = static_cast( + std::lower_bound(sorted.begin(), sorted.end(), entry.second) - + sorted.begin()); + const double dt = entry.second - medianExcluding(sorted, p); + h2_dtVsFeb_->Fill(entry.first, dt); + + // Measured on the raw dt, so an arbitrarily large slip is still counted + // even when the y-axis would bury it in the overflow bin. + const double absDt = std::abs(dt); + if (absDt > maxAbsDtSeen_) { + maxAbsDtSeen_ = absDt; + } + if (absDt > config_.dtVsFebRange) { + ++nDtOutOfRange_; + const int febId = entry.first; + if (febId >= 0 && febId < nFebIdBins()) { + if (h_dtOutOfRangePerFeb_) { + h_dtOutOfRangePerFeb_->Fill(febId); + } + dtOutOfRangeThisEvent_.push_back(febId); + } + } + } + } + + if (!timingFpgaDir_) { + return; + } + + for (const auto& [febId, fpgaMap] : hitTimes) { + for (auto itA = fpgaMap.begin(); itA != fpgaMap.end(); ++itA) { + for (auto itB = itA; itB != fpgaMap.end(); ++itB) { + const uint8_t fpgaA = itA->first; + const uint8_t fpgaB = itB->first; + const auto& hitsA = itA->second; + const auto& hitsB = itB->second; + + const uint8_t pairCode = static_cast(fpgaA * 4 + fpgaB); + const auto key = std::make_pair(febId, pairCode); + + if (h1_dtFpgaPairs_.find(key) == h1_dtFpgaPairs_.end()) { + const std::string name = + Form("dt_feb%02d_fpga%d_fpga%d", febId, fpgaA, fpgaB); + const std::string title = Form( + "#Deltat FEB %02d FPGA %d - FPGA %d;#Deltat [ns];Entries", + febId, + fpgaA, + fpgaB); + h1_dtFpgaPairs_[key] = timingFpgaDir_->make(name.c_str(), + title.c_str(), + nBinsDt_, + -config_.dtRange, + config_.dtRange); + } + + TH1F* h = h1_dtFpgaPairs_[key]; + if (fpgaA == fpgaB) { + for (std::size_t ia = 0; ia < hitsA.size(); ++ia) { + for (std::size_t ib = ia + 1; ib < hitsA.size(); ++ib) { + if (hitsA[ia].channel != hitsA[ib].channel) { + h->Fill(hitsA[ib].time_ns - hitsA[ia].time_ns); + } + } + } + } else { + for (const auto& hA : hitsA) { + for (const auto& hB : hitsB) { + h->Fill(hB.time_ns - hA.time_ns); + } + } + } + } + } + } +} + +void CRVDigiDQM::fillMicroBunchStatus(const CrvStatusCollection& crvStatus) +{ + if (!dir_) { + return; + } + + for (const auto& status : crvStatus) { + if (!status.HasROCHeader()) { + continue; + } + + const uint8_t linkID = status.GetLinkID(); + const uint32_t ubStatus = status.GetMicroBunchStatus(); + const uint64_t ewt = status.GetEventWindowTag(); + + if (g_ubStatusVsEwt_.find(linkID) == g_ubStatusVsEwt_.end()) { + const std::string name = Form("g_ubStatusVsEwt_link%d", linkID); + const std::string title = Form("MicroBunchStatus vs EWT (link %d);" + "Event window tag;MicroBunchStatus", + linkID); + TGraph* g = dir_->make(); + g->SetName(name.c_str()); + g->SetTitle(title.c_str()); + g->SetPoint(0, static_cast(ewt), static_cast(ubStatus)); + g_ubStatusVsEwt_[linkID] = g; + lastMicroBunchStatus_[linkID] = ubStatus; + continue; + } + + if (ubStatus != lastMicroBunchStatus_[linkID]) { + TGraph* g = g_ubStatusVsEwt_[linkID]; + g->SetPoint(g->GetN(), + static_cast(ewt), + static_cast(ubStatus)); + lastMicroBunchStatus_[linkID] = ubStatus; + } + } +} + +void CRVDigiDQM::persistGraph(TGraph* g) +{ + if (!dir_ || g == nullptr) { + return; + } + if (g->GetN() <= 0) { + dir_->makeAndRegister(g->GetName(), g->GetTitle()); + return; + } + dir_->makeAndRegister(g->GetName(), + g->GetTitle(), + g->GetN(), + g->GetX(), + g->GetY()); +} + +void CRVDigiDQM::scaleRateHists() +{ + if (ratesScaled_ || nEvents_ == 0) { + return; + } + const float invN = 1.0f / static_cast(nEvents_); + for (TH1F* h : h_crvDigiRatesROC_) { + if (h) { + h->Scale(invN); + } + } + if (h_crvDigiRates_) { + h_crvDigiRates_->Scale(invN); + } + if (h_crvDigisPerChannel_) { + h_crvDigisPerChannel_->Scale(invN); + } + ratesScaled_ = true; +} + +int CRVDigiDQM::nDigisOffline(std::size_t channel) const +{ + if (channel >= nDigisOffline_.size()) { + return 0; + } + return nDigisOffline_[channel]; +} + +void CRVDigiDQM::BookSectorOccupancy(const std::vector& sectorNames, + const std::vector& channelToSector, + int nBins, double lo, double hi) +{ + if (!dir_ || !h_sectorOccupancy_.empty()) { + return; + } + channelToSector_ = channelToSector; + h_sectorOccupancy_.reserve(sectorNames.size()); + for (const auto& sector : sectorNames) { + const std::string name = "crvDigisPerChannelAndEvent_CRVsector" + sector; + h_sectorOccupancy_.push_back( + dir_->make(name.c_str(), name.c_str(), nBins, lo, hi)); + } +} + +void CRVDigiDQM::fillSectorOccupancy() +{ + if (h_sectorOccupancy_.empty() || nEvents_ == 0) { + return; + } + const float invN = 1.0f / static_cast(nEvents_); + for (std::size_t channel = 0; channel < channelToSector_.size(); ++channel) { + const int sector = channelToSector_[channel]; + if (sector < 0 || + static_cast(sector) >= h_sectorOccupancy_.size()) { + continue; + } + h_sectorOccupancy_[sector]->Fill(nDigisOffline(channel) * invN); + } +} + +void CRVDigiDQM::WriteGraphs() +{ + if (!booked_ || !dir_) { + return; + } + fillSectorOccupancy(); + scaleRateHists(); + persistGraph(g_digisVsEwt_); + persistGraph(g_digisAvgVsEwt_); + for (auto& entry : g_ubStatusVsEwt_) { + persistGraph(entry.second); + } +} + +} // namespace mu2e diff --git a/DQMHelpers/src/CRVDigiDQMAnalyzer_module.cc b/DQMHelpers/src/CRVDigiDQMAnalyzer_module.cc new file mode 100644 index 0000000000..78ad87a350 --- /dev/null +++ b/DQMHelpers/src/CRVDigiDQMAnalyzer_module.cc @@ -0,0 +1,263 @@ +// Thin art analyzer that constructs, books, and fills CRVDigiDQM. +// +// Original Author: R. Mina + +#include "Offline/CosmicRayShieldGeom/inc/CosmicRayShield.hh" +#include "Offline/DataProducts/inc/CRSScintillatorBarIndex.hh" +#include "Offline/DataProducts/inc/CRVId.hh" +#include "Offline/DQMHelpers/inc/CRVDigiDQM.hh" +#include "Offline/GeometryService/inc/GeomHandle.hh" +#include "Offline/GeometryService/inc/GeometryService.hh" +#include "Offline/RecoDataProducts/inc/CrvDigi.hh" +#include "Offline/RecoDataProducts/inc/CrvStatus.hh" + +#include "art/Framework/Core/EDAnalyzer.h" +#include "art/Framework/Core/ModuleMacros.h" +#include "art/Framework/Principal/Event.h" +#include "art/Framework/Principal/Handle.h" +#include "art/Framework/Principal/Run.h" +#include "art_root_io/TFileDirectory.h" +#include "art_root_io/TFileService.h" +#include "canvas/Utilities/InputTag.h" +#include "fhiclcpp/types/Atom.h" +#include "fhiclcpp/types/Table.h" + +#include "TH1F.h" +#include "TString.h" + +#include +#include +#include +#include + +namespace mu2e { + +class CRVDigiDQMAnalyzer : public art::EDAnalyzer { +public: + struct Config { + using Name = fhicl::Name; + using Comment = fhicl::Comment; + + fhicl::Atom crvDigiTag{ + Name("crvDigiTag"), + Comment("CRV digi producer"), + art::InputTag{"CrvDigi"}}; + fhicl::Atom crvStatusTag{ + Name("crvStatusTag"), + Comment("CRV status producer"), + art::InputTag{"CrvDigi"}}; + fhicl::Atom outputTag{ + Name("outputTag"), + Comment("TFileService subdirectory; empty books in the module directory"), + ""}; + fhicl::Atom diagLevel{Name("diagLevel"), Comment("Diagnostic level"), 0}; + + fhicl::Atom nBinsDigisPerEvt{ + Name("nBinsDigisPerEvt"), Comment("Bins for h1_digisPerEvt"), 200}; + fhicl::Atom maxDigisPerEvt{ + Name("maxDigisPerEvt"), Comment("Upper edge for h1_digisPerEvt"), 4000.f}; + fhicl::Atom nBinsPeakAdc{ + Name("nBinsPeakAdc"), Comment("Bins for h1_peakAdc"), 450}; + fhicl::Atom maxPeakAdc{ + Name("maxPeakAdc"), Comment("Upper edge for h1_peakAdc"), 4500.f}; + fhicl::Atom nBinsTdc{Name("nBinsTdc"), Comment("Bins for h1_tdc"), 400}; + fhicl::Atom maxTdc{Name("maxTdc"), Comment("Upper edge for h1_tdc"), 40000.f}; + fhicl::Atom cfFraction{ + Name("cfFraction"), Comment("Constant-fraction timing threshold"), 0.20}; + fhicl::Atom dtBinSize{ + Name("dtBinSize"), Comment("CF dt histogram bin width [ns]"), 0.5f}; + fhicl::Atom dtRange{ + Name("dtRange"), Comment("CF dt histogram +/- range [ns]"), 100.f}; + fhicl::Atom dtVsFebBinSize{ + Name("dtVsFebBinSize"), Comment("dtVsFeb bin width [ns]"), 2.f}; + fhicl::Atom dtVsFebRange{ + Name("dtVsFebRange"), Comment("dtVsFeb +/- range [ns]"), 500.f}; + fhicl::Atom minAmplitude{ + Name("minAmplitude"), Comment("Minimum CF amplitude (peak-baseline)"), 10}; + fhicl::Atom avgBlockSize{ + Name("avgBlockSize"), Comment("Events per g_digisAvgVsEwt point"), 30}; + fhicl::Atom avgGraphPoints{ + Name("avgGraphPoints"), Comment("Max points in g_digisAvgVsEwt"), 1000}; + fhicl::Atom channelsWindowEwts{ + Name("channelsWindowEwts"), + Comment("EWT span for h1_channelsLastEwt"), + 50000}; + fhicl::Atom fillInclusive{ + Name("fillInclusive"), + Comment("Also fill ValCrvDigi BarId/SiPM/ADC histograms"), + true}; + fhicl::Atom fillCrvIdRates{ + Name("fillCrvIdRates"), + Comment("Book CRVId rate maps and crvDigisPerChannel"), + true}; + fhicl::Atom kppReadout{ + Name("kppReadout"), + Comment("KPP FEB-axis sizing (ROC 1-2); ROC4->ROC2 is the unpacker's job"), + true}; + fhicl::Atom fillSectorOccupancy{ + Name("fillSectorOccupancy"), + Comment("Fill per-sector crvDigisPerChannelAndEvent_* using GeometryService"), + false}; + fhicl::Atom histDigisBins{ + Name("histDigisBins"), + Comment("Bins for crvDigisPerChannelAndEvent_CRVsector*"), + 200}; + fhicl::Atom histDigisStart{ + Name("histDigisStart"), + Comment("Low edge for crvDigisPerChannelAndEvent_CRVsector*"), + 0.0}; + fhicl::Atom histDigisEnd{ + Name("histDigisEnd"), + Comment("High edge for crvDigisPerChannelAndEvent_CRVsector*"), + 0.1}; + }; + + using Parameters = art::EDAnalyzer::Table; + + explicit CRVDigiDQMAnalyzer(const Parameters& conf); + + void beginJob() override; + void beginRun(const art::Run& run) override; + void analyze(const art::Event& event) override; + void endJob() override; + +private: + static CRVDigiDQM::Config makeHelperConfig(const Config& conf); + + art::InputTag crvDigiTag_; + art::InputTag crvStatusTag_; + std::string outputTag_; + int diagLevel_; + bool fillSectorOccupancy_; + int histDigisBins_; + double histDigisStart_; + double histDigisEnd_; + CRVDigiDQM dqm_; + bool sectorMapSent_{false}; +}; + +CRVDigiDQM::Config CRVDigiDQMAnalyzer::makeHelperConfig(const Config& conf) +{ + CRVDigiDQM::Config c; + c.nBinsDigisPerEvt = conf.nBinsDigisPerEvt(); + c.maxDigisPerEvt = conf.maxDigisPerEvt(); + c.nBinsPeakAdc = conf.nBinsPeakAdc(); + c.maxPeakAdc = conf.maxPeakAdc(); + c.nBinsTdc = conf.nBinsTdc(); + c.maxTdc = conf.maxTdc(); + c.cfFraction = conf.cfFraction(); + c.dtBinSize = conf.dtBinSize(); + c.dtRange = conf.dtRange(); + c.dtVsFebBinSize = conf.dtVsFebBinSize(); + c.dtVsFebRange = conf.dtVsFebRange(); + c.minAmplitude = conf.minAmplitude(); + c.avgBlockSize = static_cast(std::max(conf.avgBlockSize(), 1)); + c.avgGraphPoints = static_cast(std::max(conf.avgGraphPoints(), 1)); + c.channelsWindowEwts = + static_cast(std::max(conf.channelsWindowEwts(), 1)); + c.fillInclusive = conf.fillInclusive(); + c.fillCrvIdRates = conf.fillCrvIdRates(); + c.kppReadout = conf.kppReadout(); + return c; +} + +CRVDigiDQMAnalyzer::CRVDigiDQMAnalyzer(const Parameters& conf) : + art::EDAnalyzer{conf}, + crvDigiTag_(conf().crvDigiTag()), + crvStatusTag_(conf().crvStatusTag()), + outputTag_(conf().outputTag()), + diagLevel_(conf().diagLevel()), + fillSectorOccupancy_(conf().fillSectorOccupancy()), + histDigisBins_(std::max(conf().histDigisBins(), 1)), + histDigisStart_(conf().histDigisStart()), + histDigisEnd_(conf().histDigisEnd()), + dqm_(makeHelperConfig(conf())) +{} + +void CRVDigiDQMAnalyzer::beginJob() +{ + art::ServiceHandle tfs; + if (outputTag_.empty()) { + // TFileService already gives each module its own directory; book into it. + art::TFileDirectory dir = *tfs; + dqm_.Book(dir); + } else { + dqm_.Book(tfs->mkdir(outputTag_)); + } +} + +void CRVDigiDQMAnalyzer::beginRun(const art::Run&) +{ + if (!fillSectorOccupancy_ || sectorMapSent_) { + return; + } + sectorMapSent_ = true; + + GeomHandle CRS; + auto const& crvSectors = CRS->getCRSScintillatorShields(); + std::vector sectorNames; + sectorNames.reserve(crvSectors.size()); + for (std::size_t i = 0; i < crvSectors.size(); ++i) { + sectorNames.emplace_back(crvSectors.at(i).name("")); + } + + auto const& crvCounters = CRS->getAllCRSScintillatorBars(); + std::vector channelToSector(crvCounters.size() * CRVId::nChanPerBar, -1); + for (std::size_t channel = 0; channel < channelToSector.size(); ++channel) { + CRSScintillatorBarIndex barIndex( + static_cast(channel / CRVId::nChanPerBar)); + channelToSector[channel] = CRS->getBar(barIndex).id().getShieldNumber(); + } + + dqm_.BookSectorOccupancy(sectorNames, channelToSector, + histDigisBins_, histDigisStart_, histDigisEnd_); +} + +void CRVDigiDQMAnalyzer::analyze(const art::Event& event) +{ + art::Handle digiHandle; + event.getByLabel(crvDigiTag_, digiHandle); + if (!digiHandle.isValid()) { + if (diagLevel_ > 1) { + std::cout << "[CRVDigiDQMAnalyzer] No CrvDigiCollection at " + << crvDigiTag_ << std::endl; + } + return; + } + + art::Handle statusHandle; + event.getByLabel(crvStatusTag_, statusHandle); + const CrvStatusCollection emptyStatus; + const CrvStatusCollection& status = + (statusHandle.isValid() && statusHandle.product() != nullptr) ? + *statusHandle : + emptyStatus; + + dqm_.Fill(*digiHandle, status); +} + +void CRVDigiDQMAnalyzer::endJob() +{ + dqm_.WriteGraphs(); + + if (diagLevel_ > 0) { + std::cout << "[CRVDigiDQMAnalyzer] Total events: " << dqm_.nEvents() + << std::endl; + std::cout << "[CRVDigiDQMAnalyzer] Total digis: " << dqm_.nDigis() + << std::endl; + std::cout << "[CRVDigiDQMAnalyzer] Active FEBs: " << dqm_.activeFEBs().size() + << std::endl; + for (const auto& [roc, febs] : dqm_.rocFEBMap()) { + std::cout << "[CRVDigiDQMAnalyzer] ROC " << static_cast(roc) << " has " + << febs.size() << " FEBs:"; + for (auto feb : febs) { + std::cout << " " << static_cast(feb); + } + std::cout << std::endl; + } + } +} + +} // namespace mu2e + +DEFINE_ART_MODULE(mu2e::CRVDigiDQMAnalyzer) diff --git a/DQMHelpers/src/CRVStatusDQM.cc b/DQMHelpers/src/CRVStatusDQM.cc new file mode 100644 index 0000000000..b22bac8181 --- /dev/null +++ b/DQMHelpers/src/CRVStatusDQM.cc @@ -0,0 +1,413 @@ +// Standalone CRV ROC-status DQM helper. +// +// Original Author: R. Mina + +#include "Offline/DQMHelpers/inc/CRVStatusDQM.hh" + +#include "TString.h" + +#include +#include +#include +#include + +namespace mu2e { + +namespace { + +const char* kErrorBitLabels[CRVStatusDQM::kNErrorBits] = { + "FEBuBMismatch", + "FEBBufferIssue", + "FEBOverflow", + "Group1Issue", + "Group2Issue", + "Group3Issue", + "uBMatchError", + "Truncation"}; + +} // namespace + +const char* CRVStatusDQM::errorBitLabel(int bitIndex) +{ + if (bitIndex < 0 || bitIndex >= kNErrorBits) { + return ""; + } + return kErrorBitLabels[bitIndex]; +} + +int CRVStatusDQM::rocBin(uint8_t dtcId, uint8_t linkId) +{ + return static_cast(dtcId) * kNLinksPerDTC + static_cast(linkId); +} + +CRVStatusDQM::CRVStatusDQM(const Config& config) : config_(config) {} + +void CRVStatusDQM::Book(art::TFileDirectory dir) +{ + dir_ = dir; + + h_nRocHeaders_ = dir.make( + "nRocHeaders", "ROC headers per event;N(ROC headers);Events", 20, -0.5, 19.5); + + h_activeFebCount_ = dir.make( + "activeFebCount", + "Active FEBs per ROC header;N(active FEBs);ROC headers", + 25, + -0.5, + 24.5); + + h_triggerCount_ = dir.make("triggerCount", + "ROC TriggerCount;TriggerCount;ROC headers", + config_.nBinsTriggerCount, + 0, + config_.maxTriggerCount); + + h_wordCount_ = dir.make("wordCount", + "ROC ControllerEventWordCount;WordCount;ROC headers", + config_.nBinsWordCount, + 0, + config_.maxWordCount); + + h_linkLatency_ = dir.make("linkLatency", + "DTC link latency;Latency;Status blocks", + config_.nBinsLatency, + 0, + config_.maxLinkLatency); + + h_errorBits_ = dir.make( + "errorBits", + "Firmware error-bit occupancy;Error bit;Counts", + kNErrorBits, + -0.5, + kNErrorBits - 0.5); + + h_errorBitsVsRoc_ = dir.make( + "errorBitsVsRoc", + "Firmware error bits vs ROC;Error bit;DTC#times6 + link ID", + kNErrorBits, + -0.5, + kNErrorBits - 0.5, + kNRocBins, + -0.5, + kNRocBins - 0.5); + + for (int i = 0; i < kNErrorBits; ++i) { + h_errorBits_->GetXaxis()->SetBinLabel(i + 1, kErrorBitLabels[i]); + h_errorBitsVsRoc_->GetXaxis()->SetBinLabel(i + 1, kErrorBitLabels[i]); + } + + h_portFlags_ = dir.make( + "portFlags", "MicroBunch port flags (bits 0-23);Port;Counts", 24, -0.5, 23.5); + + h_rocCensus_ = dir.make( + "rocCensus", + "ROC headers by DTC#times6 + link;DTC#times6 + link ID;ROC headers", + kNRocBins, + -0.5, + kNRocBins - 0.5); + + h_eventHasError_ = dir.make( + "eventHasError", + "Event has any firmware error bit;0=ok 1=error;Events", + 2, + -0.5, + 1.5); + + h_eventHasDaqError_ = dir.make( + "eventHasDaqError", + "Event has unpack DAQ error (excl. wrongSubsystemID);0=ok 1=error;Events", + 2, + -0.5, + 1.5); + + h_daqErrorCode_ = dir.make( + "daqErrorCode", + "CrvDAQerror code;Error code;Counts", + kNDaqErrorCodes, + -0.5, + kNDaqErrorCodes - 0.5); + h_daqErrorCode_->GetXaxis()->SetBinLabel(1, "unknown"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(2, "unableToGetDataBlock"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(3, "invalidPacket"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(4, "wrongSubsystemID"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(5, "errorUnpackingStatusPacket"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(6, "errorUnpackingCrvHits"); + h_daqErrorCode_->GetXaxis()->SetBinLabel(7, "byteCountMismatch"); + + h_ewtMismatch_ = dir.make("ewtMismatch", + "ROC EWT - DTC EWT;#Delta EWT;ROC headers", + config_.nBinsEwtMismatch, + -config_.maxEwtMismatch, + config_.maxEwtMismatch); + + h_errorsPerSubrun_ = dir.make( + "errorsPerSubrun", + "Events with any firmware error bit, per subrun;Events with error;Subruns", + 50, + 0, + 50); + + h_meanLatencyPerSubrun_ = dir.make( + "meanLatencyPerSubrun", + "Mean link latency per subrun;Mean latency;Subruns", + config_.nBinsLatency, + 0, + config_.maxLinkLatency); + + g_errorsVsSubrun_ = dir.make(); + g_errorsVsSubrun_->SetName("g_errorsVsSubrun"); + g_errorsVsSubrun_->SetTitle( + "Events with firmware error vs subrun;Subrun;Events with error"); + + g_meanLatencyVsSubrun_ = dir.make(); + g_meanLatencyVsSubrun_->SetName("g_meanLatencyVsSubrun"); + g_meanLatencyVsSubrun_->SetTitle( + "Mean link latency vs subrun;Subrun;Mean latency"); + + booked_ = true; +} + +TH1F* CRVStatusDQM::latencyHistFor(uint8_t dtcId, uint8_t linkId) +{ + if (!booked_ || !dir_) { + return nullptr; + } + const auto key = std::make_pair(dtcId, linkId); + auto it = h_linkLatencyByRoc_.find(key); + if (it != h_linkLatencyByRoc_.end()) { + return it->second; + } + const std::string name = + Form("linkLatency_dtc%u_roc%u", static_cast(dtcId), + static_cast(linkId)); + const std::string title = + Form("DTC %u ROC %u link latency;Latency;Status blocks", + static_cast(dtcId), + static_cast(linkId)); + TH1F* h = dir_->make(name.c_str(), + title.c_str(), + config_.nBinsLatency, + 0, + config_.maxLinkLatency); + h_linkLatencyByRoc_[key] = h; + return h; +} + +void CRVStatusDQM::Fill(const CrvStatusCollection& crvStatus) +{ + ++nEvents_; + ++nEventsThisSubrun_; + lastEventRocs_.clear(); + + int nHeadersThisEvent = 0; + bool anyErrorThisEvent = false; + + for (const auto& status : crvStatus) { + const uint8_t dtcId = status.GetDTCID(); + const uint8_t linkId = status.GetLinkID(); + const uint16_t latency = status.GetLinkLatency(); + + latencySumThisSubrun_ += latency; + ++latencyNThisSubrun_; + + if (booked_ && h_linkLatency_) { + h_linkLatency_->Fill(latency); + if (TH1F* h = latencyHistFor(dtcId, linkId)) { + h->Fill(latency); + } + } + + if (!status.HasROCHeader()) { + continue; + } + + const auto& headers = status.GetROCHeader(); + const auto& roc = headers.front(); + + ++nHeadersThisEvent; + ++nRocHeadersTotal_; + seenRocs_.insert({dtcId, linkId}); + const int ybin = rocBin(dtcId, linkId); + if (booked_ && h_rocCensus_ && ybin >= 0 && ybin < kNRocBins) { + h_rocCensus_->Fill(ybin); + } + + const std::bitset<24> activeFEBs = roc.GetActiveFEBFlags(); + const uint16_t nActive = static_cast(activeFEBs.count()); + const uint16_t trigCount = roc.TriggerCount; + const uint16_t wordCount = roc.ControllerEventWordCount; + const uint32_t ubStatus = roc.GetMicroBunchStatus(); + const uint64_t rocEwt = roc.GetEventWindowTag(); + const uint64_t dtcEwt = status.GetEventWindowTag(); + + RocSnapshot snap; + snap.dtcId = dtcId; + snap.linkId = linkId; + snap.ewt = rocEwt; + snap.triggerCount = trigCount; + snap.wordCount = wordCount; + snap.activeFebCount = nActive; + snap.linkLatency = latency; + snap.microBunchStatus = ubStatus; + lastEventRocs_.push_back(snap); + + nActiveFEBsMin_ = std::min(nActiveFEBsMin_, nActive); + nActiveFEBsMax_ = std::max(nActiveFEBsMax_, nActive); + nActiveFEBsSum_ += nActive; + ++nActiveFEBsSamples_; + + if (booked_) { + if (h_activeFebCount_) { + h_activeFebCount_->Fill(nActive); + } + if (h_triggerCount_) { + h_triggerCount_->Fill(trigCount); + } + if (h_wordCount_) { + h_wordCount_->Fill(wordCount); + } + if (h_ewtMismatch_) { + h_ewtMismatch_->Fill(static_cast(static_cast(rocEwt) - + static_cast(dtcEwt))); + } + } + + const uint32_t portFlags = ubStatus & 0x00FFFFFFu; + for (int p = 0; p < kNPortFlags; ++p) { + if ((portFlags >> p) & 1u) { + if (booked_ && h_portFlags_) { + h_portFlags_->Fill(p); + } + } + } + + for (int b = 0; b < kNErrorBits; ++b) { + if ((ubStatus >> (kErrorBitOffset + b)) & 1u) { + ++errorBitCounts_[b]; + anyErrorThisEvent = true; + if (booked_ && h_errorBits_) { + h_errorBits_->Fill(b); + } + if (booked_ && h_errorBitsVsRoc_ && ybin >= 0 && ybin < kNRocBins) { + h_errorBitsVsRoc_->Fill(b, ybin); + } + } + } + } + + if (booked_ && h_nRocHeaders_) { + h_nRocHeaders_->Fill(nHeadersThisEvent); + } + if (nHeadersThisEvent > 0) { + ++nEventsWithRocHeader_; + } + if (anyErrorThisEvent) { + ++nEventsWithAnyErrorBit_; + ++nEventsWithAnyErrorBitThisSubrun_; + } + if (booked_ && h_eventHasError_) { + h_eventHasError_->Fill(anyErrorThisEvent ? 1.f : 0.f); + } +} + +void CRVStatusDQM::Fill(const CrvStatusCollection& crvStatus, + const CrvDAQerrorCollection& crvDaqErrors) +{ + Fill(crvStatus); + fillDaqErrors(crvDaqErrors); +} + +void CRVStatusDQM::fillDaqErrors(const CrvDAQerrorCollection& crvDaqErrors) +{ + bool countedEvent = false; + for (const auto& err : crvDaqErrors) { + const int code = static_cast(err.GetErrorCode()); + if (booked_ && h_daqErrorCode_ && code >= 0 && code < kNDaqErrorCodes) { + h_daqErrorCode_->Fill(code); + } + if (err.GetErrorCode() == CrvDAQerrorCode::wrongSubsystemID) { + continue; + } + if (!countedEvent) { + ++nEventsWithDaqErrors_; + countedEvent = true; + } + } + if (booked_ && h_eventHasDaqError_) { + h_eventHasDaqError_->Fill(countedEvent ? 1.f : 0.f); + } +} + +void CRVStatusDQM::EndSubRun(int /*run*/, int subrun) +{ + if (nEventsThisSubrun_ == 0) { + return; + } + + const double meanLat = (latencyNThisSubrun_ > 0) ? + static_cast(latencySumThisSubrun_) / + static_cast(latencyNThisSubrun_) : + 0.0; + + if (booked_ && h_errorsPerSubrun_) { + h_errorsPerSubrun_->Fill(static_cast(nEventsWithAnyErrorBitThisSubrun_)); + } + if (booked_ && h_meanLatencyPerSubrun_) { + h_meanLatencyPerSubrun_->Fill(static_cast(meanLat)); + } + if (g_errorsVsSubrun_) { + g_errorsVsSubrun_->SetPoint(g_errorsVsSubrun_->GetN(), + subrun, + static_cast(nEventsWithAnyErrorBitThisSubrun_)); + } + if (g_meanLatencyVsSubrun_) { + g_meanLatencyVsSubrun_->SetPoint( + g_meanLatencyVsSubrun_->GetN(), subrun, meanLat); + } + + nEventsThisSubrun_ = 0; + nEventsWithAnyErrorBitThisSubrun_ = 0; + latencySumThisSubrun_ = 0; + latencyNThisSubrun_ = 0; +} + +void CRVStatusDQM::persistGraph(TGraph* g) +{ + if (!dir_ || g == nullptr) { + return; + } + if (g->GetN() <= 0) { + dir_->makeAndRegister(g->GetName(), g->GetTitle()); + return; + } + dir_->makeAndRegister( + g->GetName(), g->GetTitle(), g->GetN(), g->GetX(), g->GetY()); +} + +void CRVStatusDQM::WriteGraphs() +{ + if (!booked_ || !dir_) { + return; + } + persistGraph(g_errorsVsSubrun_); + persistGraph(g_meanLatencyVsSubrun_); +} + +double CRVStatusDQM::nActiveFEBsMean() const +{ + if (nActiveFEBsSamples_ == 0) { + return 0.0; + } + return static_cast(nActiveFEBsSum_) / + static_cast(nActiveFEBsSamples_); +} + +std::size_t CRVStatusDQM::errorBitCount(int bitIndex) const +{ + if (bitIndex < 0 || bitIndex >= kNErrorBits) { + return 0; + } + return errorBitCounts_[bitIndex]; +} + +} // namespace mu2e diff --git a/DQMHelpers/src/CRVStatusDQMAnalyzer_module.cc b/DQMHelpers/src/CRVStatusDQMAnalyzer_module.cc new file mode 100644 index 0000000000..775d6ce7d3 --- /dev/null +++ b/DQMHelpers/src/CRVStatusDQMAnalyzer_module.cc @@ -0,0 +1,182 @@ +// Thin art analyzer that constructs, books, and fills CRVStatusDQM. +// +// Original Author: R. Mina + +#include "Offline/DQMHelpers/inc/CRVStatusDQM.hh" +#include "Offline/RecoDataProducts/inc/CrvDAQerror.hh" +#include "Offline/RecoDataProducts/inc/CrvStatus.hh" + +#include "art/Framework/Core/EDAnalyzer.h" +#include "art/Framework/Core/ModuleMacros.h" +#include "art/Framework/Principal/Event.h" +#include "art/Framework/Principal/Handle.h" +#include "art/Framework/Principal/SubRun.h" +#include "art_root_io/TFileDirectory.h" +#include "art_root_io/TFileService.h" +#include "canvas/Utilities/InputTag.h" +#include "fhiclcpp/types/Atom.h" +#include "fhiclcpp/types/Table.h" + +#include +#include +#include + +namespace mu2e { + +class CRVStatusDQMAnalyzer : public art::EDAnalyzer { +public: + struct Config { + using Name = fhicl::Name; + using Comment = fhicl::Comment; + + fhicl::Atom crvStatusTag{ + Name("crvStatusTag"), + Comment("CRV status producer"), + art::InputTag{"CrvDigi"}}; + fhicl::Atom crvDaqErrorTag{ + Name("crvDaqErrorTag"), + Comment("CRV DAQ-error producer"), + art::InputTag{"CrvDigi"}}; + fhicl::Atom outputTag{ + Name("outputTag"), + Comment("TFileService subdirectory; empty books in the module directory"), + ""}; + fhicl::Atom diagLevel{Name("diagLevel"), Comment("Diagnostic level"), 0}; + + fhicl::Atom nBinsLatency{ + Name("nBinsLatency"), Comment("Bins for linkLatency"), 1024}; + fhicl::Atom maxLinkLatency{ + Name("maxLinkLatency"), Comment("Upper edge for linkLatency"), 4096.f}; + fhicl::Atom nBinsTriggerCount{ + Name("nBinsTriggerCount"), Comment("Bins for triggerCount"), 256}; + fhicl::Atom maxTriggerCount{ + Name("maxTriggerCount"), Comment("Upper edge for triggerCount"), 65535.f}; + fhicl::Atom nBinsWordCount{ + Name("nBinsWordCount"), Comment("Bins for wordCount"), 256}; + fhicl::Atom maxWordCount{ + Name("maxWordCount"), Comment("Upper edge for wordCount"), 65535.f}; + fhicl::Atom nBinsEwtMismatch{ + Name("nBinsEwtMismatch"), Comment("Bins for ewtMismatch"), 201}; + fhicl::Atom maxEwtMismatch{ + Name("maxEwtMismatch"), Comment("Abs range for ewtMismatch"), 100.f}; + }; + + using Parameters = art::EDAnalyzer::Table; + + explicit CRVStatusDQMAnalyzer(const Parameters& conf); + + void beginJob() override; + void analyze(const art::Event& event) override; + void endSubRun(const art::SubRun& subrun) override; + void endJob() override; + +private: + static CRVStatusDQM::Config makeHelperConfig(const Config& conf); + + art::InputTag crvStatusTag_; + art::InputTag crvDaqErrorTag_; + std::string outputTag_; + int diagLevel_; + CRVStatusDQM dqm_; +}; + +CRVStatusDQM::Config CRVStatusDQMAnalyzer::makeHelperConfig(const Config& conf) +{ + CRVStatusDQM::Config c; + c.nBinsLatency = std::max(conf.nBinsLatency(), 1); + c.maxLinkLatency = conf.maxLinkLatency(); + c.nBinsTriggerCount = std::max(conf.nBinsTriggerCount(), 1); + c.maxTriggerCount = conf.maxTriggerCount(); + c.nBinsWordCount = std::max(conf.nBinsWordCount(), 1); + c.maxWordCount = conf.maxWordCount(); + c.nBinsEwtMismatch = std::max(conf.nBinsEwtMismatch(), 1); + c.maxEwtMismatch = conf.maxEwtMismatch(); + return c; +} + +CRVStatusDQMAnalyzer::CRVStatusDQMAnalyzer(const Parameters& conf) : + art::EDAnalyzer{conf}, + crvStatusTag_(conf().crvStatusTag()), + crvDaqErrorTag_(conf().crvDaqErrorTag()), + outputTag_(conf().outputTag()), + diagLevel_(conf().diagLevel()), + dqm_(makeHelperConfig(conf())) +{} + +void CRVStatusDQMAnalyzer::beginJob() +{ + art::ServiceHandle tfs; + if (outputTag_.empty()) { + // TFileService already gives each module its own directory; book into it. + art::TFileDirectory dir = *tfs; + dqm_.Book(dir); + } else { + dqm_.Book(tfs->mkdir(outputTag_)); + } +} + +void CRVStatusDQMAnalyzer::analyze(const art::Event& event) +{ + art::Handle statusHandle; + event.getByLabel(crvStatusTag_, statusHandle); + const CrvStatusCollection emptyStatus; + const CrvStatusCollection& status = + (statusHandle.isValid() && statusHandle.product() != nullptr) ? + *statusHandle : + emptyStatus; + + art::Handle daqHandle; + event.getByLabel(crvDaqErrorTag_, daqHandle); + if (daqHandle.isValid() && daqHandle.product() != nullptr) { + dqm_.Fill(status, *daqHandle); + } else { + dqm_.Fill(status); + } + + if (diagLevel_ > 1) { + std::cout << "[CRVStatusDQMAnalyzer] " << event.id() + << " nStatus=" << status.size() + << " nRocSnap=" << dqm_.lastEventRocs().size() << std::endl; + } +} + +void CRVStatusDQMAnalyzer::endSubRun(const art::SubRun& subrun) +{ + dqm_.EndSubRun(static_cast(subrun.run()), + static_cast(subrun.subRun())); +} + +void CRVStatusDQMAnalyzer::endJob() +{ + dqm_.WriteGraphs(); + + if (diagLevel_ > 0) { + std::cout << "[CRVStatusDQMAnalyzer] Total events: " << dqm_.nEvents() + << std::endl; + std::cout << "[CRVStatusDQMAnalyzer] Events with ROC header: " + << dqm_.nEventsWithRocHeader() << std::endl; + std::cout << "[CRVStatusDQMAnalyzer] Events with firmware error bit: " + << dqm_.nEventsWithAnyErrorBit() << std::endl; + std::cout << "[CRVStatusDQMAnalyzer] Events with DAQ unpack error: " + << dqm_.nEventsWithDaqErrors() << std::endl; + std::cout << "[CRVStatusDQMAnalyzer] Distinct ROCs: " + << dqm_.seenRocs().size() << std::endl; + if (dqm_.nActiveFEBsMin() != 65535) { + std::cout << "[CRVStatusDQMAnalyzer] Active FEBs min/mean/max: " + << dqm_.nActiveFEBsMin() << " / " << dqm_.nActiveFEBsMean() + << " / " << dqm_.nActiveFEBsMax() << std::endl; + } + for (int b = 0; b < CRVStatusDQM::kNErrorBits; ++b) { + std::cout << "[CRVStatusDQMAnalyzer] " << CRVStatusDQM::errorBitLabel(b) + << ": " << dqm_.errorBitCount(b) << std::endl; + } + for (const auto& roc : dqm_.seenRocs()) { + std::cout << "[CRVStatusDQMAnalyzer] seen DTC " << static_cast(roc.first) + << " link " << static_cast(roc.second) << std::endl; + } + } +} + +} // namespace mu2e + +DEFINE_ART_MODULE(mu2e::CRVStatusDQMAnalyzer) diff --git a/DQMHelpers/src/SConscript b/DQMHelpers/src/SConscript new file mode 100644 index 0000000000..3b98969204 --- /dev/null +++ b/DQMHelpers/src/SConscript @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# Original author R. Mina. + +Import('env') +Import('mu2e_helper') + +helper = mu2e_helper(env) + +rootlibs = env['ROOTLIBS'] + +# CrvStatus.hh pulls artdaq CRVROCStatusPacketFEBII, whose accessors both +# helper .cc files call; scons LIBS are not transitive, so list it here. +mainlib = helper.make_mainlib([ + 'mu2e_RecoDataProducts', + 'mu2e_DataProducts', + 'artdaq-core-mu2e_Overlays', + 'art_Framework_Services_Registry', + 'art_root_io_tfile_support', + 'art_root_io_TFileService', + 'art_Utilities', + 'canvas', + 'MF_MessageLogger', + 'cetlib', + 'cetlib_except', + rootlibs, +]) + +helper.make_plugins([ + mainlib, + 'mu2e_RecoDataProducts', + 'mu2e_DataProducts', + 'artdaq-core-mu2e_Overlays', + 'mu2e_GeometryService', + 'mu2e_CosmicRayShieldGeom', + 'mu2e_GeomPrimitives', + 'art_Framework_Core', + 'art_Framework_Principal', + 'art_Framework_Services_Registry', + 'art_root_io_tfile_support', + 'art_root_io_TFileService', + 'art_Persistency_Common', + 'art_Persistency_Provenance', + 'art_Utilities', + 'canvas', + 'MF_MessageLogger', + 'fhiclcpp', + 'fhiclcpp_types', + 'tbb', + 'cetlib', + 'cetlib_except', + rootlibs, + 'boost_filesystem', +]) + +# Local Variables: +# mode:python +# End: diff --git a/RecoDataProducts/inc/CrvStatus.hh b/RecoDataProducts/inc/CrvStatus.hh index ba09955fc7..9fd30f46e9 100644 --- a/RecoDataProducts/inc/CrvStatus.hh +++ b/RecoDataProducts/inc/CrvStatus.hh @@ -65,6 +65,7 @@ namespace mu2e uint8_t GetLinkStatus() const {return _linkStatus;} uint16_t GetLinkLatency() const {return _linkLatency;} std::vector &GetROCHeader() {return _rocHeader;} + const std::vector &GetROCHeader() const {return _rocHeader;} bool HasROCHeader() const {return !_rocHeader.empty();} // --- MicroBunchStatus accessors (32-bit CRV ROC status word) ---