[17.0] Extend convertToRaw to support streamer files

HLTrigger/Tools/README.md

@@ -1,26 +1,32 @@
 # convertToRaw
 
-Convert RAW data stored in one or more EDM .root files into the .raw file used as input by the HLT.
 
-```
-usage: convertToRaw [-h] [-o PATH] [-f EVENTS] [-l EVENTS] [--one-file-per-lumi] FILES [FILES ...]
+Convert raw data stored into one or more EDM files format (.root files) or Streamer format (.dat files) into the DAQ
+format (.raw files) used as input by the HLT.
+
+The default behaviour is to process a single luminosity section at a time, in order to support luminosity sections split
+across multiple files and a limit on the number of events in each lumisection.
+If neither of these features is needed (i.e. if lumisections are not split, and all events should be converted) the `-1`
+or `--one-file-per-lumi` can be used to process all data with a single job, speeding up the conversion considerably.
 
-Convert RAW data from .root format to .raw format.
+```
+usage: convertToRaw [-h] [-s TAG] [-o PATH] [-f EVENTS] [-l EVENTS] [-r [RUN:LUMI-RUN:LUMI]] [-v] [-1] FILES [FILES ...]
 
 positional arguments:
-  FILES                 input files in .root format
+  FILES                 input files in .root or .dat format
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
+  -s TAG, --source TAG  name of the FEDRawDataCollection to be repacked into raw format (default: rawDataCollector)
   -o PATH, --output PATH
-                        base path to store the output files; subdirectories based on the run number are automatically created (default: )
+                        base path to store the output files; subdirectories based on the run number are automatically created (default: /home/fwyzard/src/cmssw/HLTrigger/Tools)
   -f EVENTS, --events_per_file EVENTS
-                        split the output into files with at most EVENTS events (default: 50)
+                        split the output into files with at most EVENTS events (default: 100)
   -l EVENTS, --events_per_lumi EVENTS
-                        process at most EVENTS events in each lumisection (default: 11650)
-  --one-file-per-lumi   assume that lumisections are not split across files (and disable --events_per_lumi) (default: False)
+                        process at most EVENTS events in each lumisection (default: 11655)
+  -r [RUN:LUMI-RUN:LUMI], --range [RUN:LUMI-RUN:LUMI]
+                        process only the runs and lumisections in the given range (default: all)
+  -v, --verbose         print additional information while processing the input files (default: False)
+  -1, --one-file-per-lumi
+                        assume that lumisections are not split across files (and disable --events_per_lumi) (default: False)
 ```
-
-The default behaviour is to process a single luminosity section at a time, in order to support luminosity sections split across multiple files and a limit on the number of events in each lumisection.
-
-If neither of these features is needed (_i.e._ if lumisections are not split, and all events should be converted) the `--one-file-per-lumi` can be used to process all data with a single job, speeding up the conversion considerably.

HLTrigger/Tools/python/streamerToRaw.py

@@ -0,0 +1,134 @@
+# Convert the RAW data from streamer .dat files into DAQ .raw format
+#
+# usage: cmsRun $CMSSW_RELEASE_BASE/HLTrigger/Tools/python/streamerToRaw.py \
+#           inputFiles=/store/path/file.dat[,/store/path/file.dat,...] \
+#           runNumber=NNNNNN \
+#           [lumiNumber=NNNN] \
+#           [eventsPerFile=50] \
+#           [eventsPerLumi=11650] \
+#           [rawDataCollection=rawDataCollector] \
+#           [outputPath=output_directory]
+#
+# The output files will appear as output_directory/runNNNNNN/runNNNNNN_lumiNNNN_indexNNNNNN.raw .
+
+import sys
+import os
+import FWCore.ParameterSet.Config as cms
+import FWCore.ParameterSet.VarParsing as VarParsing
+
+process = cms.Process("FAKE")
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(-1)                                 # to be overwritten after parsing the command line options
+)
+
+process.source = cms.Source("NewEventStreamFileReader",
+    fileNames = cms.untracked.vstring()                             # to be overwritten after parsing the command line options
+)
+
+from EventFilter.Utilities.EvFDaqDirector_cfi import EvFDaqDirector as _EvFDaqDirector
+process.EvFDaqDirector = _EvFDaqDirector.clone(
+    baseDir = "",                                                   # to be overwritten after parsing the command line options
+    buBaseDir = "",                                                 # to be overwritten after parsing the command line options
+    runNumber = 0                                                   # to be overwritten after parsing the command line options
+)
+
+process.writer = cms.OutputModule("RawStreamFileWriterForBU",
+    source = cms.InputTag('rawDataCollector'),                      # to be overwritten after parsing the command line options
+    numEventsPerFile = cms.uint32(0)                                # to be overwritten after parsing the command line options
+)
+
+process.endpath = cms.EndPath(process.writer)
+
+process.load('FWCore.MessageService.MessageLogger_cfi')
+process.MessageLogger.cerr.FwkReport.reportEvery = 0                # to be overwritten after parsing the command line options
+
+# parse command line options
+options = VarParsing.VarParsing ('python')
+for name in 'filePrepend', 'maxEvents', 'outputFile', 'secondaryOutputFile', 'section', 'tag', 'storePrepend', 'totalSections':
+    del options._register[name]
+    del options._beenSet[name]
+    del options._info[name]
+    del options._types[name]
+    if name in options._singletons:
+        del options._singletons[name]
+    if name in options._lists:
+        del options._lists[name]
+    if name in options._noCommaSplit:
+        del options._noCommaSplit[name]
+    if name in options._noDefaultClear:
+        del options._noDefaultClear[name]
+
+
+options.register('runNumber',
+                 0,
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Run number to use")
+
+options.register('lumiNumber',
+                 None,
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Luminosity section number to use")
+
+options.register('eventsPerLumi',
+                 11650,
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Number of events in the given luminosity section to process")
+
+options.register('eventsPerFile',
+                 50,
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Split the output into files with at most this number of events")
+
+options.register('rawDataCollection',
+                 'rawDataCollector',
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.string,
+                 "FEDRawDataCollection to be repacked into RAW format")
+
+options.register('outputPath',
+                 os.getcwd(),
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.string,
+                 "Output directory for the FED RAW data files")
+
+options.parseArguments()
+
+# check that the option values are valide
+if options.runNumber <= 0:
+    sys.stderr.write('Invalid run number\n')
+    sys.exit(1)
+
+if options.lumiNumber is not None and options.lumiNumber <= 0:
+    sys.stderr.write('Invalid luminosity section number\n')
+    sys.exit(1)
+
+if options.eventsPerLumi == 0 or options.eventsPerLumi < -1:
+    sys.stderr.write('Invalid number of events per luminosity section\n')
+    sys.exit(1)
+
+if options.eventsPerFile <= 0:
+    sys.stderr.write('Invalid number of events per output file\n')
+    sys.exit(1)
+
+# configure the job based on the command line options
+process.source.fileNames = options.inputFiles
+if options.lumiNumber is not None:
+    # process only one lumisection
+    process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange('%d:%d' % (options.runNumber, options.lumiNumber))
+    process.maxEvents.input = options.eventsPerLumi
+process.EvFDaqDirector.runNumber = options.runNumber
+process.EvFDaqDirector.baseDir = options.outputPath
+process.EvFDaqDirector.buBaseDir = options.outputPath
+process.writer.source = options.rawDataCollection
+process.writer.numEventsPerFile = options.eventsPerFile
+process.MessageLogger.cerr.FwkReport.reportEvery = options.eventsPerFile
+
+# create the output directory, if it does not exist
+outputRunPath = f'{options.outputPath}/run{options.runNumber:06d}'
+os.makedirs(outputRunPath, exist_ok=True)
+open(f'{outputRunPath}/fu.lock', 'w').close()