feat: detector changed hook

.github/workflows/ci.yml

@@ -271,14 +271,16 @@ jobs:
         id:
           - kpp
           - eb-ep
+          - eb-en
           - eb-eg
           - eb-epc
           - eb-enc
           - eb-eftpi
         include:
-          # run all tests on ubuntu
+          # run selected tests on ubuntu
           - { id: kpp,      cmd: ./run-advanced-tests.sh }
           - { id: eb-ep,    cmd: ./run-eb-tests.sh -100 electronproton   }
+          - { id: eb-en,    cmd: ./run-eb-tests.sh -100 electronneutron  }
           - { id: eb-eg,    cmd: ./run-eb-tests.sh -100 electrongamma    }
           - { id: eb-epc,   cmd: ./run-eb-tests.sh -100 electronprotonC  }
           - { id: eb-enc,   cmd: ./run-eb-tests.sh -100 electronneutronC }

common-tools/clas-decay-tools/src/main/java/org/jlab/clas/decay/analysis/Analysis.java

@@ -67,7 +67,7 @@ public void setFieldsConfig(String fieldsConfig) {
 
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         Reader reader ;
         this.FieldsConfig = this.getFieldsConfig();
         if (event.hasBank("RUN::config") == false) {
@@ -250,5 +250,7 @@ public double[] getBeamSpot(DataEvent event, IndexedTable beamPos) {
         }
         return xybeam;
     }
+
+    public void detectorChanged(int run) {}
 
 }

common-tools/clas-io/src/main/java/org/jlab/utils/JsonUtils.java

@@ -113,9 +113,7 @@ public static JsonObject Map2Json(Map<String,Object> map) {
                 ret.add(topKey,Map2Json((Map)entry.getValue()));
             }
             else {
-                if(entry.getValue() != null) {
-                    ret.add(topKey, entry.getValue().toString());
-                }
+                ret.add(topKey, entry.getValue() == null ? null : entry.getValue().toString());
             }
         }
         return ret;

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java

@@ -282,7 +282,7 @@ public void init(){
     public void processEvent(DataEvent event){
         for(Map.Entry<String,ReconstructionEngine> engine : this.processorEngines.entrySet()){
             try {
-                engine.getValue().filterEvent(event);
+                engine.getValue().processDataEvent(event);
             } catch (Exception e){
                 LOGGER.log(Level.SEVERE, "[Exception] >>>>> engine : {0}\n\n", engine.getKey());
                 e.printStackTrace();

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java

@@ -2,6 +2,7 @@
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -60,14 +61,22 @@ public abstract class ReconstructionEngine implements Engine {
     volatile boolean dropOutputBanks = false;
     private final Set<String> outputBanks = new HashSet<>();
 
+    private volatile List<Integer> runNumbers = new ArrayList<>();
+
     private boolean ignoreInvalidRunNumbers = true;
 
+    private int runNumberOverride = -1;
+
     volatile long triggerMask = 0xFFFFFFFFFFFFFFFFL;
 
-    String             engineName        = "UnknownEngine";
-    String             engineAuthor      = "N.T.";
-    String             engineVersion     = "0.0";
-    String             engineDescription = "CLARA Engine";
+    String engineName        = "UnknownEngine";
+    String engineAuthor      = "N.T.";
+    String engineVersion     = "0.0";
+    String engineDescription = "CLARA Engine";
+
+    abstract public boolean processDataEventUser(DataEvent event);
+    abstract public boolean init();
+    abstract public void detectorChanged(int runNumber);
 
     public ReconstructionEngine(String name, String author, String version){
         engineName    = name;
@@ -108,9 +117,6 @@ protected RawBank getRawBankReader(String bankName, OrderType... order) {
         return new RawDataBank(bankName, order);
     }
 
-    abstract public boolean processDataEvent(DataEvent event);
-    abstract public boolean init();
-
     /**
      * Use a map just to avoid name clash in ConstantsManager.
      * @param tables map of table names to #indices
@@ -187,6 +193,9 @@ public EngineData configure(EngineData ed) {
           engineDictionary = new SchemaFactory();
       LOGGER.log(Level.FINEST,"--- engine configuration is called " + this.getDescription());
       try {
+          if (this.getEngineConfigString("runNumberOverride")!=null) {
+              this.runNumberOverride = Integer.valueOf(this.getEngineConfigString("runNumberOverride"));
+          }
           if (this.getEngineConfigString("rawBankGroup")!=null) {
               this.rawBankOrders = RawBank.getFilterGroup(this.getEngineConfigString("rawBankGroup"));
           }
@@ -227,8 +236,7 @@ public EngineData configure(EngineData ed) {
     }
 
     protected String getStringConfigParameter(String jsonString,                                             
-            String key)  throws Exception {
-        Object js;
+                                              String key)  throws Exception {
         String variation = "";
         try {
             JSONObject base = new JSONObject(jsonString);
@@ -238,13 +246,6 @@ protected String getStringConfigParameter(String jsonString,
             } else {
                 LOGGER.log(Level.FINE,"[JSON]" + this.getName() + " **** warning **** does not contain key = " + key);
             }
-            /*
-            js = base.get(key);
-            if (js instanceof String) {
-                return (String) js;
-            } else {
-                throw new Exception("JSONObject[" +  "] not a string.");
-            }*/
         } catch (JSONException e) {
             throw new Exception(e.getMessage());
         }
@@ -345,17 +346,22 @@ public void dropBanks(DataEvent event) {
             }
         }
     }
-
-    public boolean checkRunNumber(DataEvent event) {
-        if (!this.ignoreInvalidRunNumbers) return true;
-        int run = 0;
-        if (event.hasBank("RUN::config")) {
-            run = event.getBank("RUN::config").getInt("run",0);
+
+    public synchronized boolean checkRunNumber(DataEvent event) {
+        int r = runNumberOverride;
+        if (r <= 0 && event.hasBank("RUN::config")) {
+            r = event.getBank("RUN::config").getInt("run",0);
         }
-        return run>0;
+        if (r > 0) {
+            if (this.runNumbers.isEmpty() || r != this.runNumbers.get(this.runNumbers.size()-1)) {
+                this.runNumbers.add(r);
+                this.detectorChanged(r);
+            }
+        }
+        return !this.ignoreInvalidRunNumbers || r>0;
     }
 
-    public void filterEvent(DataEvent dataEvent) {
+    public void processDataEvent(DataEvent dataEvent) {
         if (!this.wroteConfig) {
             this.wroteConfig = true;
             JsonUtils.extend(dataEvent, CONFIG_BANK_NAME, "json", this.generateConfig());
@@ -365,7 +371,7 @@ public void filterEvent(DataEvent dataEvent) {
         }
         if(this.applyTriggerMask(dataEvent)) {
             if (this.checkRunNumber(dataEvent)) {
-                this.processDataEvent(dataEvent);
+                this.processDataEventUser(dataEvent);
             }
         }        
     }
@@ -404,7 +410,7 @@ public EngineData execute(EngineData input) {
             }
 
             try {
-                this.filterEvent(dataEventHipo);
+                this.processDataEvent(dataEventHipo);
                 output.setData(mt, dataEventHipo.getHipoEvent());
             } catch (Exception e) {
                 String msg = String.format("Error processing input event%n%n%s", ClaraUtil.reportException(e));
@@ -519,16 +525,21 @@ public Reco(){
             super("a","b","c");
         }
         @Override
-        public boolean processDataEvent(DataEvent event) {
-            throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+        public boolean processDataEventUser(DataEvent event) {
+            throw new UnsupportedOperationException("Not supported yet.");
         }
 
         @Override
         public boolean init() {
-            throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+            throw new UnsupportedOperationException("Not supported yet.");
         }
-
-}
+
+        @Override
+        public void detectorChanged(int runNumber) {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+    }
+
     public static void main(String[] args){
         System.setProperty("CLAS12DIR", "/Users/gavalian/Work/Software/project-3a.0.0/Distribution/clas12-offline-software/coatjava");
         try {
@@ -549,7 +560,6 @@ public static void main(String[] args){
                     "\"timestamp\":333\n" +
                     "}";
             System.out.println(json);
-            //json = "{ \"ccdb\":{\"run\":10,\"variation\":\"default\"}, \"variation\":\"cosmic\"}";
             Reco reco = new Reco();
             String variation =  reco.getStringConfigParameter(json, "variation");
             System.out.println(" Variation : " + variation);

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/UberEngine.java

@@ -30,7 +30,7 @@ protected void add(ReconstructionEngine... e) {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         throw new RuntimeException("UberEngine does not implement processDataEvent.");
     }
 
@@ -39,9 +39,9 @@ public boolean processDataEvent(DataEvent event) {
      * @param event 
      */
     @Override
-    public final void filterEvent(DataEvent event) {
+    public final void processDataEvent(DataEvent event) {
         for (ReconstructionEngine e : engines) 
-            e.filterEvent(event);
+            e.processDataEvent(event);
     }
 
     /**
@@ -55,6 +55,16 @@ public final boolean init() {
             if (!e.init()) ret = false;
         return ret;
     }
+
+    /**
+     * Run all engines' dettectorChanged methods.
+     * @param runNumber
+     */ 
+    @Override
+    public final void detectorChanged(int runNumber) {
+        for (ReconstructionEngine e : engines)
+            e.detectorChanged(runNumber);
+    }
 
     /**
      * Run all engines' configure methods.

common-tools/swim-tools/src/main/java/org/jlab/clas/swimtools/MagFieldsEngine.java

@@ -162,7 +162,7 @@ private void loadTables() {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         DataBank bank = event.getBank("RUN::config");
         // Load the constants
         // -------------------
@@ -195,4 +195,6 @@ public boolean init() {
         return true;
     }
 
+    @Override
+    public void detectorChanged(int runNumber) {}
 }

etc/services/mc-ai.yaml

@@ -18,15 +18,14 @@ services:
     name: FTEB
   - class: org.jlab.service.raster.RasterEngine
     name: RASTER
-# DC clustering: nominal algoritm, filled the cluster bank for ai and conventional tracking
+  - class: org.jlab.service.ai.DCDenoiseEngine
+    name: DCDN    
   - class: org.jlab.service.dc.DCHBClustering
     name: DCCR
-# This is where the AI plugin would go.  THe AI code reads the HB clusters.  
-  - class: org.jlab.service.mltn.MLTDEngine
-    name: MLTD
-# DC using clustering on AI hits: NN assisted HB tracking
-  - class: org.jlab.service.dc.DCHBPostClusterAI
-    name: DCHAI
+  - class: org.jlab.service.ai.DCClsComboEngine
+    name: DCCC
+  - class: org.jlab.service.dc.DCHBTrackingAI
+    name: DCHTAI
   - class: org.jlab.service.ftof.FTOFHBEngine
     name: FTOFHB
   - class: org.jlab.service.ec.ECEngine
@@ -64,11 +63,10 @@ services:
   - class: org.jlab.calibration.service.CalibBanksEngine
     name: CALIB
 configuration:
-#  global:
-#    variation: rga_fall2018_mc
-#    timestamp: 12/31/2020-00:00:00
-#    dcMinistagger: "NOTONREFWIRE"
-#    dcFeedthroughs: "OFF"
+  global:
+    variation: mc2026
+    dcMinistagger: "NOTONREFWIRE"
+    dcFeedthroughs: "OFF"
   io-services:
     writer:
       schema_dir: dst
@@ -77,9 +75,7 @@ configuration:
     MAGFIELDS:
       magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat
       magfieldTorusMap: Symm_torus_r2501_phi16_z251_24Apr2018.dat
-    MLTD:
-      run: "6302"
-    DCHAI:
+    DCHTAI:
       outputBankPrefix: "HB"
     RTPC:
       rtpcSimulation: "true"

etc/services/mc-aicv.yaml

@@ -18,18 +18,16 @@ services:
     name: FTEB
   - class: org.jlab.service.raster.RasterEngine
     name: RASTER
-# DC clustering: nominal algoritm, filled the cluster bank for ai and conventional tracking
+#  - class: org.jlab.service.ai.DCDenoiseEngine
+#    name: DCDN    
   - class: org.jlab.service.dc.DCHBClustering
     name: DCCR
-# This is where the AI plugin would go.  THe AI code reads the HB clusters.  
-  - class: org.jlab.service.mltn.MLTDEngine
-    name: MLTD
-# DC using clustering: regular HB tracking
+  - class: org.jlab.service.ai.DCClsComboEngine
+    name: DCCC
   - class: org.jlab.service.dc.DCHBPostClusterConv
     name: DCHB
-# DC using clustering on AI hits: NN assisted HB tracking
-  - class: org.jlab.service.dc.DCHBPostClusterAI
-    name: DCHAI
+  - class: org.jlab.service.dc.DCHBTrackingAI
+    name: DCHTAI
   - class: org.jlab.service.ftof.FTOFHBEngine
     name: FTOFHB
   - class: org.jlab.service.ec.ECEngine
@@ -75,10 +73,10 @@ services:
   - class: org.jlab.calibration.service.CalibBanksEngine
     name: CALIB
 configuration:
-#  global:
-#    variation: rga_fall2018_mc
-#    dcMinistagger: "NOTONREFWIRE"
-#    dcFeedthroughs: "OFF"
+  global:
+    variation: mc2026
+    dcMinistagger: "NOTONREFWIRE"
+    dcFeedthroughs: "OFF"
   io-services:
     writer:
       schema_dir: mc-aicv

etc/services/mc-cv.yaml

@@ -62,10 +62,10 @@ services:
   - class: org.jlab.calibration.service.CalibBanksEngine
     name: CALIB
 configuration:
-#  global:
-#    variation: rga_fall2018_mc
-#    dcMinistagger: "NOTONREFWIRE"
-#    dcFeedthroughs: "OFF"
+  global:
+    variation: mc2026
+    dcMinistagger: "NOTONREFWIRE"
+    dcFeedthroughs: "OFF"
   io-services:
     writer:
       schema_dir: dst