From c3db7f03e41585ccfd86ffe08e38e4838dda6240 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 16 Dec 2020 13:20:46 +0100 Subject: [PATCH 1/3] DPL: move ROOT file reading completely to the plugin This way we can have the plugin depend on JAlien and not require everyone does. --- Framework/AnalysisSupport/CMakeLists.txt | 1 + .../src/AODJAlienReaderHelpers.cxx | 323 ++++++++++++++++++ .../src/AODJAlienReaderHelpers.h | 28 ++ Framework/AnalysisSupport/src/Plugin.cxx | 4 +- .../Core/include/Framework/AODReaderHelpers.h | 42 --- Framework/Core/src/AODReaderHelpers.cxx | 176 ---------- .../Core/src/AnalysisDataModelHelpers.cxx | 26 -- Framework/Core/src/AnalysisDataModelHelpers.h | 1 - 8 files changed, 354 insertions(+), 247 deletions(-) create mode 100644 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx create mode 100644 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt index 68dc3fc6f9b2b..0f87506224545 100644 --- a/Framework/AnalysisSupport/CMakeLists.txt +++ b/Framework/AnalysisSupport/CMakeLists.txt @@ -12,5 +12,6 @@ # Drop this once we move to GCC 8.2+ o2_add_library(FrameworkAnalysisSupport SOURCES src/Plugin.cxx + src/AODJAlienReaderHelpers.cxx PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src PUBLIC_LINK_LIBRARIES O2::Framework) diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx new file mode 100644 index 0000000000000..acffd4f5de7b8 --- /dev/null +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -0,0 +1,323 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "AODJAlienReaderHelpers.h" +#include "Framework/TableTreeHelpers.h" +#include "Framework/AnalysisHelpers.h" +#include "Framework/RootTableBuilderHelpers.h" +#include "Framework/AlgorithmSpec.h" +#include "Framework/ConfigParamRegistry.h" +#include "Framework/ControlService.h" +#include "Framework/CallbackService.h" +#include "Framework/EndOfStreamContext.h" +#include "Framework/DeviceSpec.h" +#include "Framework/RawDeviceService.h" +#include "Framework/DataSpecUtils.h" +#include "Framework/DataInputDirector.h" +#include "Framework/SourceInfoHeader.h" +#include "Framework/ChannelInfo.h" +#include "Framework/Logger.h" + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +using namespace o2; +using namespace o2::aod; + +struct RuntimeWatchdog { + int numberTimeFrames; + uint64_t startTime; + uint64_t lastTime; + double runTime; + uint64_t runTimeLimit; + + RuntimeWatchdog(Long64_t limit) + { + numberTimeFrames = -1; + startTime = uv_hrtime(); + lastTime = startTime; + runTime = 0.; + runTimeLimit = limit; + } + + bool update() + { + numberTimeFrames++; + if (runTimeLimit <= 0) { + return true; + } + + auto nowTime = uv_hrtime(); + + // time spent to process the time frame + double time_spent = numberTimeFrames < 1 ? (double)(nowTime - lastTime) / 1.E9 : 0.; + runTime += time_spent; + lastTime = nowTime; + + return ((double)(lastTime - startTime) / 1.E9 + runTime / (numberTimeFrames + 1)) < runTimeLimit; + } + + void printOut() + { + LOGP(INFO, "RuntimeWatchdog"); + LOGP(INFO, " run time limit: {}", runTimeLimit); + LOGP(INFO, " number of time frames: {}", numberTimeFrames); + LOGP(INFO, " estimated run time per time frame: {}", (numberTimeFrames >= 0) ? runTime / (numberTimeFrames + 1) : 0.); + LOGP(INFO, " estimated total run time: {}", (double)(lastTime - startTime) / 1.E9 + ((numberTimeFrames >= 0) ? runTime / (numberTimeFrames + 1) : 0.)); + } +}; + +template +static constexpr auto columnNamesTrait(framework::pack) +{ + return std::vector{C::columnLabel()...}; +} + +std::vector getColumnNames(header::DataHeader dh) +{ + auto description = std::string(dh.dataDescription.str); + auto origin = std::string(dh.dataOrigin.str); + + // get column names + // AOD / RN2 + if (origin == "AOD") { + if (description == "TRACK:PAR") { + return columnNamesTrait(typename StoredTracksMetadata::table_t::persistent_columns_t{}); + } else if (description == "TRACK:PARCOV") { + return columnNamesTrait(typename StoredTracksCovMetadata::table_t::persistent_columns_t{}); + } else if (description == "TRACK:EXTRA") { + return columnNamesTrait(typename TracksExtraMetadata::table_t::persistent_columns_t{}); + } + } + + // default: column names = {} + return std::vector({}); +} + +using o2::monitoring::Metric; +using o2::monitoring::Monitoring; +using o2::monitoring::tags::Key; +using o2::monitoring::tags::Value; + +namespace o2::framework::readers +{ +auto setEOSCallback(InitContext& ic) +{ + ic.services().get().set(CallbackService::Id::EndOfStream, + [](EndOfStreamContext& eosc) { + auto& control = eosc.services().get(); + control.endOfStream(); + control.readyToQuit(QuitRequest::Me); + }); +} + +template +static inline auto extractTypedOriginal(ProcessingContext& pc) +{ + ///FIXME: this should be done in invokeProcess() as some of the originals may be compound tables + return O{pc.inputs().get(aod::MetadataTrait::metadata::tableLabel())->asArrowTable()}; +} + +template +static inline auto extractOriginalsTuple(framework::pack, ProcessingContext& pc) +{ + return std::make_tuple(extractTypedOriginal(pc)...); +} + +AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback() +{ + auto callback = AlgorithmSpec{adaptStateful([](ConfigParamRegistry const& options, + DeviceSpec const& spec, + Monitoring& monitoring) { + monitoring.send(Metric{(uint64_t)0, "arrow-bytes-created"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.send(Metric{(uint64_t)0, "arrow-messages-created"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.send(Metric{(uint64_t)0, "arrow-bytes-destroyed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.send(Metric{(uint64_t)0, "arrow-messages-destroyed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.flushBuffer(); + + if (!options.isSet("aod-file")) { + LOGP(ERROR, "No input file defined!"); + throw std::runtime_error("Processing is stopped!"); + } + + auto filename = options.get("aod-file"); + + // create a DataInputDirector + auto didir = std::make_shared(filename); + if (options.isSet("aod-reader-json")) { + auto jsonFile = options.get("aod-reader-json"); + if (!didir->readJson(jsonFile)) { + LOGP(ERROR, "Check the JSON document! Can not be properly parsed!"); + } + } + + // get the run time watchdog + auto* watchdog = new RuntimeWatchdog(options.get("time-limit")); + + // selected the TFN input and + // create list of requested tables + header::DataHeader TFNumberHeader; + std::vector requestedTables; + std::vector routes(spec.outputs); + for (auto route : routes) { + if (DataSpecUtils::partialMatch(route.matcher, header::DataOrigin("TFN"))) { + auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher); + TFNumberHeader = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec); + } else { + requestedTables.emplace_back(route); + } + } + + auto fileCounter = std::make_shared(0); + auto numTF = std::make_shared(-1); + return adaptStateless([TFNumberHeader, + requestedTables, + fileCounter, + numTF, + watchdog, + didir](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) { + // check if RuntimeLimit is reached + if (!watchdog->update()) { + LOGP(INFO, "Run time exceeds run time limit of {} seconds!", watchdog->runTimeLimit); + LOGP(INFO, "Stopping reader {} after time frame {}.", device.inputTimesliceId, watchdog->numberTimeFrames - 1); + monitoring.flushBuffer(); + didir->closeInputFiles(); + control.endOfStream(); + control.readyToQuit(QuitRequest::Me); + return; + } + + // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId + // the TF to read is numTF + assert(device.inputTimesliceId < device.maxInputTimeslices); + uint64_t timeFrameNumber = 0; + int fcnt = (*fileCounter * device.maxInputTimeslices) + device.inputTimesliceId; + int ntf = *numTF + 1; + monitoring.send(Metric{(uint64_t)ntf, "tf-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + static int currentFileCounter = -1; + static int filesProcessed = 0; + if (currentFileCounter != *fileCounter) { + currentFileCounter = *fileCounter; + monitoring.send(Metric{(uint64_t)++filesProcessed, "files-opened"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + } + + // loop over requested tables + TTree* tr = nullptr; + bool first = true; + static size_t totalSizeUncompressed = 0; + static size_t totalSizeCompressed = 0; + static size_t totalReadCalls = 0; + + for (auto route : requestedTables) { + + // create header + auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher); + auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec); + + // create a TreeToTable object + auto info = didir->getFileFolder(dh, fcnt, ntf); + size_t before = 0; + if (info.file) { + info.file->GetReadCalls(); + } + tr = didir->getDataTree(dh, fcnt, ntf); + if (!tr) { + if (first) { + // check if there is a next file to read + fcnt += device.maxInputTimeslices; + if (didir->atEnd(fcnt)) { + LOGP(INFO, "No input files left to read for reader {}!", device.inputTimesliceId); + didir->closeInputFiles(); + control.endOfStream(); + control.readyToQuit(QuitRequest::Me); + return; + } + // get first folder of next file + ntf = 0; + tr = didir->getDataTree(dh, fcnt, ntf); + if (!tr) { + LOGP(FATAL, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf); + throw std::runtime_error("Processing is stopped!"); + } + } else { + LOGP(FATAL, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf); + throw std::runtime_error("Processing is stopped!"); + } + } + TTreePerfStats ps("ioperf", tr); + + if (first) { + timeFrameNumber = didir->getTimeFrameNumber(dh, fcnt, ntf); + auto o = Output(TFNumberHeader); + outputs.make(o) = timeFrameNumber; + } + + // create table output + auto o = Output(dh); + auto& t2t = outputs.make(o); + + // add branches to read + // fill the table + + auto colnames = getColumnNames(dh); + if (colnames.size() == 0) { + totalSizeCompressed += tr->GetZipBytes(); + totalSizeUncompressed += tr->GetTotBytes(); + t2t.addAllColumns(tr); + } else { + for (auto& colname : colnames) { + TBranch* branch = tr->GetBranch(colname.c_str()); + totalSizeCompressed += branch->GetZipBytes("*"); + totalSizeUncompressed += branch->GetTotBytes("*"); + t2t.addColumn(colname.c_str()); + } + } + t2t.fill(tr); + if (info.file) { + totalReadCalls += info.file->GetReadCalls() - before; + static std::string currentFileRead = ""; + std::string nextFileRead = info.file->GetPath(); + if (currentFileRead != nextFileRead) { + currentFileRead = nextFileRead; + monitoring.send(Metric{currentFileRead, "aod-file-read-path"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + } + } + monitoring.send(Metric{(double)ps.GetReadCalls(), "aod-tree-read-calls"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + delete tr; + + first = false; + } + monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.send(Metric{(uint64_t)totalSizeCompressed / 1000, "aod-bytes-read-compressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + monitoring.send(Metric{(uint64_t)totalReadCalls, "aod-total-read-calls"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); + + // save file number and time frame + *fileCounter = (fcnt - device.inputTimesliceId) / device.maxInputTimeslices; + *numTF = ntf; + }); + })}; + + return callback; +} + +} // namespace o2::framework::readers diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h new file mode 100644 index 0000000000000..8ef579bd68edf --- /dev/null +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h @@ -0,0 +1,28 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_FRAMEWORK_AODJALIENREADERHELPERS_H_ +#define O2_FRAMEWORK_AODJALIENREADERHELPERS_H_ + +#include "Framework/TableBuilder.h" +#include "Framework/AlgorithmSpec.h" +#include "Framework/Logger.h" +#include + +namespace o2::framework::readers +{ + +struct AODJAlienReaderHelpers { + static AlgorithmSpec rootFileReaderCallback(); +}; + +} // namespace o2::framework::readers + +#endif // O2_FRAMEWORK_AODREADERHELPERS_H_ diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index 54d4bf5f84d00..1fafab25a956b 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -9,12 +9,12 @@ // or submit itself to any jurisdiction. #include "Framework/Plugins.h" #include "Framework/AlgorithmSpec.h" -#include "Framework/AODReaderHelpers.h" +#include "AODJAlienReaderHelpers.h" struct ROOTFileReader : o2::framework::AlgorithmPlugin { o2::framework::AlgorithmSpec create() override { - return o2::framework::readers::AODReaderHelpers::rootFileReaderCallback(); + return o2::framework::readers::AODJAlienReaderHelpers::rootFileReaderCallback(); } }; diff --git a/Framework/Core/include/Framework/AODReaderHelpers.h b/Framework/Core/include/Framework/AODReaderHelpers.h index f428f46bce6f5..4290f2cccba4b 100644 --- a/Framework/Core/include/Framework/AODReaderHelpers.h +++ b/Framework/Core/include/Framework/AODReaderHelpers.h @@ -19,48 +19,6 @@ namespace o2::framework::readers { -struct RuntimeWatchdog { - int numberTimeFrames; - uint64_t startTime; - uint64_t lastTime; - double runTime; - uint64_t runTimeLimit; - - RuntimeWatchdog(Long64_t limit) - { - numberTimeFrames = -1; - startTime = uv_hrtime(); - lastTime = startTime; - runTime = 0.; - runTimeLimit = limit; - } - - bool update() - { - numberTimeFrames++; - if (runTimeLimit <= 0) { - return true; - } - - auto nowTime = uv_hrtime(); - - // time spent to process the time frame - double time_spent = numberTimeFrames < 1 ? (double)(nowTime - lastTime) / 1.E9 : 0.; - runTime += time_spent; - lastTime = nowTime; - - return ((double)(lastTime - startTime) / 1.E9 + runTime / (numberTimeFrames + 1)) < runTimeLimit; - } - - void printOut() - { - LOGP(INFO, "RuntimeWatchdog"); - LOGP(INFO, " run time limit: {}", runTimeLimit); - LOGP(INFO, " number of time frames: {}", numberTimeFrames); - LOGP(INFO, " estimated run time per time frame: {}", (numberTimeFrames >= 0) ? runTime / (numberTimeFrames + 1) : 0.); - LOGP(INFO, " estimated total run time: {}", (double)(lastTime - startTime) / 1.E9 + ((numberTimeFrames >= 0) ? runTime / (numberTimeFrames + 1) : 0.)); - } -}; struct AODReaderHelpers { static AlgorithmSpec rootFileReaderCallback(); diff --git a/Framework/Core/src/AODReaderHelpers.cxx b/Framework/Core/src/AODReaderHelpers.cxx index a563578485fd9..266ac4c048814 100644 --- a/Framework/Core/src/AODReaderHelpers.cxx +++ b/Framework/Core/src/AODReaderHelpers.cxx @@ -175,180 +175,4 @@ AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector reques }}; } -AlgorithmSpec AODReaderHelpers::rootFileReaderCallback() -{ - auto callback = AlgorithmSpec{adaptStateful([](ConfigParamRegistry const& options, - DeviceSpec const& spec, - Monitoring& monitoring) { - monitoring.send(Metric{(uint64_t)0, "arrow-bytes-created"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.send(Metric{(uint64_t)0, "arrow-messages-created"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.send(Metric{(uint64_t)0, "arrow-bytes-destroyed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.send(Metric{(uint64_t)0, "arrow-messages-destroyed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.flushBuffer(); - - if (!options.isSet("aod-file")) { - LOGP(ERROR, "No input file defined!"); - throw std::runtime_error("Processing is stopped!"); - } - - auto filename = options.get("aod-file"); - - // create a DataInputDirector - auto didir = std::make_shared(filename); - if (options.isSet("aod-reader-json")) { - auto jsonFile = options.get("aod-reader-json"); - if (!didir->readJson(jsonFile)) { - LOGP(ERROR, "Check the JSON document! Can not be properly parsed!"); - } - } - - // get the run time watchdog - auto* watchdog = new RuntimeWatchdog(options.get("time-limit")); - - // selected the TFN input and - // create list of requested tables - header::DataHeader TFNumberHeader; - std::vector requestedTables; - std::vector routes(spec.outputs); - for (auto route : routes) { - if (DataSpecUtils::partialMatch(route.matcher, header::DataOrigin("TFN"))) { - auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher); - TFNumberHeader = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec); - } else { - requestedTables.emplace_back(route); - } - } - - auto fileCounter = std::make_shared(0); - auto numTF = std::make_shared(-1); - return adaptStateless([TFNumberHeader, - requestedTables, - fileCounter, - numTF, - watchdog, - didir](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) { - // check if RuntimeLimit is reached - if (!watchdog->update()) { - LOGP(INFO, "Run time exceeds run time limit of {} seconds!", watchdog->runTimeLimit); - LOGP(INFO, "Stopping reader {} after time frame {}.", device.inputTimesliceId, watchdog->numberTimeFrames - 1); - monitoring.flushBuffer(); - didir->closeInputFiles(); - control.endOfStream(); - control.readyToQuit(QuitRequest::Me); - return; - } - - // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId - // the TF to read is numTF - assert(device.inputTimesliceId < device.maxInputTimeslices); - uint64_t timeFrameNumber = 0; - int fcnt = (*fileCounter * device.maxInputTimeslices) + device.inputTimesliceId; - int ntf = *numTF + 1; - monitoring.send(Metric{(uint64_t)ntf, "tf-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - static int currentFileCounter = -1; - static int filesProcessed = 0; - if (currentFileCounter != *fileCounter) { - currentFileCounter = *fileCounter; - monitoring.send(Metric{(uint64_t)++filesProcessed, "files-opened"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - } - - // loop over requested tables - TTree* tr = nullptr; - bool first = true; - static size_t totalSizeUncompressed = 0; - static size_t totalSizeCompressed = 0; - static size_t totalReadCalls = 0; - - for (auto route : requestedTables) { - - // create header - auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher); - auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec); - - // create a TreeToTable object - auto info = didir->getFileFolder(dh, fcnt, ntf); - size_t before = 0; - if (info.file) { - info.file->GetReadCalls(); - } - tr = didir->getDataTree(dh, fcnt, ntf); - if (!tr) { - if (first) { - // check if there is a next file to read - fcnt += device.maxInputTimeslices; - if (didir->atEnd(fcnt)) { - LOGP(INFO, "No input files left to read for reader {}!", device.inputTimesliceId); - didir->closeInputFiles(); - control.endOfStream(); - control.readyToQuit(QuitRequest::Me); - return; - } - // get first folder of next file - ntf = 0; - tr = didir->getDataTree(dh, fcnt, ntf); - if (!tr) { - LOGP(FATAL, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf); - throw std::runtime_error("Processing is stopped!"); - } - } else { - LOGP(FATAL, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf); - throw std::runtime_error("Processing is stopped!"); - } - } - TTreePerfStats ps("ioperf", tr); - - if (first) { - timeFrameNumber = didir->getTimeFrameNumber(dh, fcnt, ntf); - auto o = Output(TFNumberHeader); - outputs.make(o) = timeFrameNumber; - } - - // create table output - auto o = Output(dh); - auto& t2t = outputs.make(o); - - // add branches to read - // fill the table - - auto colnames = aod::datamodel::getColumnNames(dh); - if (colnames.size() == 0) { - totalSizeCompressed += tr->GetZipBytes(); - totalSizeUncompressed += tr->GetTotBytes(); - t2t.addAllColumns(tr); - } else { - for (auto& colname : colnames) { - TBranch* branch = tr->GetBranch(colname.c_str()); - totalSizeCompressed += branch->GetZipBytes("*"); - totalSizeUncompressed += branch->GetTotBytes("*"); - t2t.addColumn(colname.c_str()); - } - } - t2t.fill(tr); - if (info.file) { - totalReadCalls += info.file->GetReadCalls() - before; - static std::string currentFileRead = ""; - std::string nextFileRead = info.file->GetPath(); - if (currentFileRead != nextFileRead) { - currentFileRead = nextFileRead; - monitoring.send(Metric{currentFileRead, "aod-file-read-path"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - } - } - monitoring.send(Metric{(double)ps.GetReadCalls(), "aod-tree-read-calls"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - delete tr; - - first = false; - } - monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.send(Metric{(uint64_t)totalSizeCompressed / 1000, "aod-bytes-read-compressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - monitoring.send(Metric{(uint64_t)totalReadCalls, "aod-total-read-calls"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); - - // save file number and time frame - *fileCounter = (fcnt - device.inputTimesliceId) / device.maxInputTimeslices; - *numTF = ntf; - }); - })}; - - return callback; -} - } // namespace o2::framework::readers diff --git a/Framework/Core/src/AnalysisDataModelHelpers.cxx b/Framework/Core/src/AnalysisDataModelHelpers.cxx index b089ebe8f68d0..8a254e70fdcb5 100644 --- a/Framework/Core/src/AnalysisDataModelHelpers.cxx +++ b/Framework/Core/src/AnalysisDataModelHelpers.cxx @@ -40,31 +40,5 @@ std::string getTreeName(header::DataHeader dh) return treeName; } -template -static constexpr auto columnNamesTrait(framework::pack) -{ - return std::vector{C::columnLabel()...}; -} - -std::vector getColumnNames(header::DataHeader dh) -{ - auto description = std::string(dh.dataDescription.str); - auto origin = std::string(dh.dataOrigin.str); - - // get column names - // AOD / RN2 - if (origin == "AOD") { - if (description == "TRACK:PAR") { - return columnNamesTrait(typename StoredTracksMetadata::table_t::persistent_columns_t{}); - } else if (description == "TRACK:PARCOV") { - return columnNamesTrait(typename StoredTracksCovMetadata::table_t::persistent_columns_t{}); - } else if (description == "TRACK:EXTRA") { - return columnNamesTrait(typename TracksExtraMetadata::table_t::persistent_columns_t{}); - } - } - - // default: column names = {} - return std::vector({}); -} } // namespace o2::aod::datamodel diff --git a/Framework/Core/src/AnalysisDataModelHelpers.h b/Framework/Core/src/AnalysisDataModelHelpers.h index 59f69782c845c..074303e869b8d 100644 --- a/Framework/Core/src/AnalysisDataModelHelpers.h +++ b/Framework/Core/src/AnalysisDataModelHelpers.h @@ -16,6 +16,5 @@ namespace o2::aod::datamodel { std::string getTreeName(header::DataHeader dh); -std::vector getColumnNames(header::DataHeader dh); } // namespace o2::aod::datamodel #endif // O2_FRAMEWORK_ANALYSISDATAMODELHELPERS_H_ From 28f4af817bd6736d4d390debbf025fd560cf26fd Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 16 Dec 2020 13:32:19 +0100 Subject: [PATCH 2/3] DPL: add (optional) direct dependency on JAliEn to AnalysisSupport This is the groundwork to be able to access TJAlienFile from the reader (without requiring everyone to depend on JAlien). --- Framework/AnalysisSupport/CMakeLists.txt | 7 +- .../src/AODJAlienReaderHelpers.cxx | 12 +- dependencies/FindJAlienROOT.cmake | 33 +++ dependencies/FindXRootD.cmake | 236 ++++++++++++++++++ dependencies/O2Dependencies.cmake | 6 + 5 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 dependencies/FindJAlienROOT.cmake create mode 100644 dependencies/FindXRootD.cmake diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt index 0f87506224545..ceed68342364c 100644 --- a/Framework/AnalysisSupport/CMakeLists.txt +++ b/Framework/AnalysisSupport/CMakeLists.txt @@ -10,8 +10,13 @@ # Given GCC 7.3 does not provide std::filesystem we use Boost instead # Drop this once we move to GCC 8.2+ + +if(TARGET XRootD::Client) + set(EXTRA_TARGETS XRootD::Client JAliEn::JAliEn) +endif() + o2_add_library(FrameworkAnalysisSupport SOURCES src/Plugin.cxx src/AODJAlienReaderHelpers.cxx PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src - PUBLIC_LINK_LIBRARIES O2::Framework) + PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS}) diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index acffd4f5de7b8..bb97107189eac 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -28,6 +28,9 @@ #include #include +#if __has_include() +#include +#endif #include #include #include @@ -237,9 +240,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback() // create a TreeToTable object auto info = didir->getFileFolder(dh, fcnt, ntf); size_t before = 0; - if (info.file) { - info.file->GetReadCalls(); - } tr = didir->getDataTree(dh, fcnt, ntf); if (!tr) { if (first) { @@ -299,6 +299,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback() std::string nextFileRead = info.file->GetPath(); if (currentFileRead != nextFileRead) { currentFileRead = nextFileRead; +#if __has_include() + auto alienFile = dynamic_cast(info.file); + if (alienFile) { + /// FIXME: get the JAlien stats + } +#endif monitoring.send(Metric{currentFileRead, "aod-file-read-path"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL)); } } diff --git a/dependencies/FindJAlienROOT.cmake b/dependencies/FindJAlienROOT.cmake new file mode 100644 index 0000000000000..b48e90b40a9b4 --- /dev/null +++ b/dependencies/FindJAlienROOT.cmake @@ -0,0 +1,33 @@ +# Copyright CERN and copyright holders of ALICE O2. This software is distributed +# under the terms of the GNU General Public License v3 (GPL Version 3), copied +# verbatim in the file "COPYING". +# +# See http://alice-o2.web.cern.ch/license for full licensing information. +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. +find_path(JALIEN_ROOT_INCLUDE_DIR TJAlienFile.h PATH_SUFFIXES include + PATHS + ${JALIEN_ROOT_ROOT}) + +find_library(JAliEnRoot_LIB JAliEnROOT PATHS ${JALIEN_ROOT_ROOT}/lib) + +if(NOT JALIEN_ROOT_INCLUDE_DIR) + set(JAliEnROOT_FOUND FALSE) + return() +endif() + +set(JAliEnROOT_FOUND TRUE) + +if(NOT TARGET JAliEn::JAliEn) + get_filename_component(libdir ${JAliEnRoot_LIB} DIRECTORY) + add_library(JAliEn::JAliEn INTERFACE IMPORTED) + set_target_properties(JAliEn::JAliEn PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${JALIEN_ROOT_INCLUDE_DIR} + INTERFACE_LINK_LIBRARIES ${JAliEnRoot_LIB} + INTERFACE_LINK_DIRECTORIES ${libdir} + ) +endif() + +mark_as_advanced(JALIEN_ROOT_INCLUDE_DIR) diff --git a/dependencies/FindXRootD.cmake b/dependencies/FindXRootD.cmake new file mode 100644 index 0000000000000..23167369eb781 --- /dev/null +++ b/dependencies/FindXRootD.cmake @@ -0,0 +1,236 @@ +################################################################################ +# Module for locating XRootD. +# +# XROOTD_FOUND +# Indicates whether the library has been found. +# +# XROOTD_INCLUDE_DIRS +# Specifies XRootD include directory. +# +# XROOTD_LIBRARIES +# Specifies XRootD libraries that should be passed to target_link_libararies. +# +# XROOTD__LIBRARIES +# Specifies the libraries of a specific +# +# XROOTD__FOUND +# Indicates whether the specified was found. +# +# List of components: CLIENT, UTILS, SERVER, POSIX, HTTP and SSI +################################################################################ + + +################################################################################ +# Set XRootD include paths +################################################################################ +FIND_PATH( XROOTD_INCLUDE_DIRS XrdVersion.hh + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES include/xrootd + PATHS /opt/xrootd +) + +IF( NOT "${XROOTD_INCLUDE_DIRS}" STREQUAL "XROOTD_INCLUDE_DIRS-NOTFOUND" ) + SET( XROOTD_FOUND TRUE ) +ENDIF() + +IF( NOT XROOTD_FOUND ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD client libs +# - libXrdCl +################################################################################ +FIND_LIBRARY( XROOTD_CLIENT_LIBRARIES XrdCl + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_CLIENT_LIBRARIES}" STREQUAL "XROOTD_CLIENT_LIBRARIES-NOTFOUND" ) + SET( XROOTD_CLIENT_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_CLIENT_LIBRARIES} ) +ENDIF() + +IF( XRootD_FIND_REQUIRED_CLIENT AND NOT XROOTD_CLIENT_FOUND ) + MESSAGE( "XRootD client required but not found!" ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_CLIENT_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD utils libs +# - libXrdUtils +################################################################################ +FIND_LIBRARY( XROOTD_UTILS_LIBRARIES XrdUtils + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_UTILS_LIBRARIES}" STREQUAL "XROOTD_UTILS_LIBRARIES-NOTFOUND" ) + SET( XROOTD_UTILS_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_UTILS_LIBRARIES} ) +ENDIF() + +IF( XRootD_FIND_REQUIRED_UTILS AND NOT XROOTD_UTILS_FOUND ) + MESSAGE( "XRootD utils required but not found!" ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_UTILS_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD server libs +# - libXrdServer +################################################################################ +FIND_LIBRARY( XROOTD_SERVER_LIBRARIES XrdServer + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_SERVER_LIBRARIES}" STREQUAL "XROOTD_SERVER_LIBRARIES-NOTFOUND" ) + SET( XROOTD_SERVER_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_SERVER_LIBRARIES} ) +ENDIF() + +IF( XRootD_FIND_REQUIRED_SERVER AND NOT XROOTD_SERVER_FOUND ) + MESSAGE( "XRootD server required but not found!" ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_SERVER_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD posix libs +# - libXrdPosix +# - libXrdPosixPreload +################################################################################ +FIND_LIBRARY( XROOTD_POSIX_LIBRARY XrdPosix + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +FIND_LIBRARY( XROOTD_POSIX_PRELOAD_LIBRARY XrdPosixPreload + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_POSIX_LIBRARY}" STREQUAL "XROOTD_POSIX_LIBRARY-NOTFOUND" ) + IF( NOT "${XROOTD_POSIX_PRELOAD_LIBRARY}" STREQUAL "XROOTD_POSIX_PRELOAD_LIBRARY-NOTFOUND" ) + SET( XROOTD_POSIX_LIBRARIES ${XROOTD_POSIX_LIBRARY} ${XROOTD_POSIX_PRELOAD_LIBRARY} ) + SET( XROOTD_POSIX_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_POSIX_LIBRARIES} ) + ENDIF() +ENDIF() + +IF( XRootD_FIND_REQUIRED_POSIX AND NOT XROOTD_POSIX_FOUND ) + MESSAGE( "XRootD posix required but not found!" ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_POSIX_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD HTTP (XrdHttp) libs +# - libXrdHtppUtils +################################################################################ +FIND_LIBRARY( XROOTD_HTTP_LIBRARIES XrdHttpUtils + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_HTTP_LIBRARIES}" STREQUAL "XROOTD_HTTP_LIBRARIES-NOTFOUND" ) + SET( XROOTD_HTTP_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_HTTP_LIBRARIES} ) +ENDIF() + +IF( XRootD_FIND_REQUIRED_HTTP AND NOT XROOTD_HTTP_FOUND ) + MESSAGE( "XRootD http required but not found!" ) + LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_HTTP_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# XRootD SSI libs +# - XrdSsiLib +# - XrdSsiShMap +################################################################################ +FIND_LIBRARY( XROOTD_SSI_LIBRARY XrdSsiLib + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +FIND_LIBRARY( XROOTD_SSI_SHMAP_LIBRARY XrdSsiShMap + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd + PATH_SUFFIXES lib lib64 +) + +IF( NOT "${XROOTD_SSI_LIBRARY}" STREQUAL "XROOTD_SSI_LIBRARY-NOTFOUND" ) + IF( NOT "${XROOTD_SSI_SHMAP_LIBRARY}" STREQUAL "XROOTD_SSI_SHMAP_LIBRARY-NOTFOUND" ) + SET( XROOTD_SSI_LIBRARIES ${XROOTD_SSI_LIBRARY} ${XROOTD_SSI_SHMAP_LIBRARY} ) + SET( XROOTD_SSI_FOUND TRUE ) + LIST( APPEND XROOTD_LIBRARIES ${XROOTD_SSI_LIBRARIES} ) + ENDIF() +ENDIF() + +IF( XRootD_FIND_REQUIRED_SSI AND NOT XROOTD_SSI_FOUND ) + MESSAGE( "XRootD ssi required but not found!" ) + LIST (APPEND _XROOTD_MISSING_COMPONENTS XROOTD_SSI_FOUND ) + UNSET( XROOTD_FOUND ) +ENDIF() + +################################################################################ +# Set up the XRootD find module +################################################################################ + +IF( XRootD_FIND_REQUIRED ) + INCLUDE( FindPackageHandleStandardArgs ) + FIND_PACKAGE_HANDLE_STANDARD_ARGS( XRootD + REQUIRED_VARS XROOTD_INCLUDE_DIRS ${_XROOTD_MISSING_COMPONENTS} + ) +ENDIF() + + +if (XROOTD_CLIENT_FOUND) +if(NOT TARGET XRootD::Client) + get_filename_component(libdir ${XROOTD_CLIENT_LIBRARIES} DIRECTORY) + add_library(XRootD::Client INTERFACE IMPORTED) + set_target_properties(XRootD::Client PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${XROOTD_INCLUDE_DIRS} + INTERFACE_LINK_LIBRARIES ${XROOTD_CLIENT_LIBRARIES} + INTERFACE_LINK_DIRECTORIES ${libdir} + ) +endif() +endif() diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake index 93cc13d58df3e..b1e751a46ff79 100644 --- a/dependencies/O2Dependencies.cmake +++ b/dependencies/O2Dependencies.cmake @@ -103,6 +103,12 @@ set_package_properties(RapidJSON PROPERTIES TYPE REQUIRED) find_package(CURL) set_package_properties(CURL PROPERTIES TYPE REQUIRED) +find_package(JAliEnROOT MODULE) +set_package_properties(JAliEnROOT PROPERTIES TYPE RECOMMENDED) + +find_package(XRootD MODULE) +set_package_properties(XRootD PROPERTIES TYPE RECOMMENDED) + # MC specific packages message(STATUS "Input BUILD_SIMULATION=${BUILD_SIMULATION}") include("${CMAKE_CURRENT_LIST_DIR}/O2SimulationDependencies.cmake") From 937c961bdd4a84f1bc52058844828fa116807970 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 15 Dec 2020 21:42:00 +0100 Subject: [PATCH 3/3] Do not compile OpenGL examples on macOS --- Framework/Core/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 06f5de71e4740..63c7e46765808 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -258,12 +258,14 @@ endforeach() # #####################################################@ if (TARGET AliceO2::DebugGUI) +if (NOT APPLE) set (DEBUG_GUI_TESTS_WORKFLOW CustomGUIGL CustomGUISokol SimpleTracksED ) endif() +endif() foreach(w BoostSerializedProcessing @@ -307,10 +309,12 @@ if (TARGET AliceO2::DebugGUI) # environment assertion fired X11: The DISPLAY environment variable is missing # glfw-3.2.1/src/window.c:579: glfwGetFramebufferSize: Assertion `window != # ((void *)0)' failed. +if(NOT APPLE) set_property(TEST test_Framework_test_SimpleTracksED PROPERTY DISABLED TRUE) set_property(TEST test_Framework_test_CustomGUIGL PROPERTY DISABLED TRUE) set_property(TEST test_Framework_test_CustomGUISokol PROPERTY DISABLED TRUE) endif() +endif() # TODO: investigate the problem and re-enable set_property(TEST test_Framework_test_BoostSerializedProcessing