Fixes in entropy compression memory management

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h

@@ -775,7 +775,8 @@ void EncodedBlocks<H, N, W>::encode(const S_IT srcBegin,     // iterator begin o
   // case 3: message where entropy coding should be applied
   if (opt == Metadata::OptStore::EENCODE) {
     // build symbol statistics
-    constexpr size_t SizeEstMargin = 10 * 1024;
+    constexpr size_t SizeEstMarginAbs = 10 * 1024;
+    constexpr float SizeEstMarginRel = 1.05;
     const o2::rans::LiteralEncoder64<STYP>* encoder = reinterpret_cast<const o2::rans::LiteralEncoder64<STYP>*>(encoderExt);
     std::unique_ptr<o2::rans::LiteralEncoder64<STYP>> encoderLoc;
     std::unique_ptr<o2::rans::FrequencyTable> frequencies = nullptr;
@@ -791,7 +792,7 @@ void EncodedBlocks<H, N, W>::encode(const S_IT srcBegin,     // iterator begin o
     // estimate size of encode buffer
     int dataSize = rans::calculateMaxBufferSize(messageLength, encoder->getAlphabetRangeBits(), sizeof(STYP)); // size in bytes
     // preliminary expansion of storage based on dict size + estimated size of encode buffer
-    dataSize = SizeEstMargin + dataSize / sizeof(W) + (sizeof(STYP) < sizeof(W)); // size in words of output stream
+    dataSize = SizeEstMarginAbs + int(SizeEstMarginRel * (dataSize / sizeof(W))) + (sizeof(STYP) < sizeof(W)); // size in words of output stream
     expandStorage(dictSize + dataSize);
     //store dictionary first
     if (dictSize) {
@@ -800,7 +801,9 @@ void EncodedBlocks<H, N, W>::encode(const S_IT srcBegin,     // iterator begin o
     // vector of incompressible literal symbols
     std::vector<STYP> literals;
     // directly encode source message into block buffer.
-    const auto encodedMessageEnd = encoder->process(bl->getCreateData(), bl->getCreateData() + dataSize, srcBegin, srcEnd, literals);
+    auto blIn = bl->getCreateData();
+    auto frSize = bl->registry->getFreeSize(); // note: "this" might be not valid after expandStorage call!!!
+    const auto encodedMessageEnd = encoder->process(blIn, blIn + frSize, srcBegin, srcEnd, literals);
     dataSize = encodedMessageEnd - bl->getData();
     bl->setNData(dataSize);
     bl->realignBlock();

Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h

@@ -72,7 +72,7 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const TriggerRecord>& trigData,
   CTFHelper helper(trigData, cluData);
 
   // book output size with some margin
-  auto szIni = sizeof(CTFHeader) + helper.getSize() / 4; // will be autoexpanded if needed
+  auto szIni = sizeof(CTFHeader) + helper.getSize() * 2. / 3; // will be autoexpanded if needed
   buff.resize(szIni);
 
   auto ec = CTF::create(buff);

Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h

@@ -72,7 +72,7 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const TriggerRecord>& trigData,
   CTFHelper helper(trigData, cellData);
 
   // book output size with some margin
-  auto szIni = sizeof(CTFHeader) + helper.getSize() / 4; // will be autoexpanded if needed
+  auto szIni = sizeof(CTFHeader) + helper.getSize() * 2. / 3; // will be autoexpanded if needed
   buff.resize(szIni);
 
   auto ec = CTF::create(buff);

Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx

@@ -182,7 +182,7 @@ size_t CTFCoder::estimateCompressedSize(const CompressedClusters& cc)
   sz += ESTSIZE(cc.pattMap,      CTF::BLCpattMap     );
 
   // clang-format on
-
+  sz *= 2. / 3; // if needed, will be autoexpanded
   LOG(INFO) << "Estimated output size is " << sz << " bytes";
   return sz;
 }

Detectors/MUON/MID/CTF/include/MIDCTF/CTFCoder.h

@@ -73,7 +73,7 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const ROFRecord>& rofData, cons
   CTFHelper helper(rofData, colData);
 
   // book output size with some margin
-  auto szIni = sizeof(CTFHeader) + helper.getSize() / 4; // will be autoexpanded if needed
+  auto szIni = sizeof(CTFHeader) + helper.getSize() * 2. / 3; // will be autoexpanded if needed
   buff.resize(szIni);
 
   auto ec = CTF::create(buff);

Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFCoder.h

@@ -72,7 +72,7 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const TriggerRecord>& trigData,
   CTFHelper helper(trigData, cellData);
 
   // book output size with some margin
-  auto szIni = sizeof(CTFHeader) + helper.getSize() / 4; // will be autoexpanded if needed
+  auto szIni = sizeof(CTFHeader) + helper.getSize() * 2. / 3; // will be autoexpanded if needed
   buff.resize(szIni);
 
   auto ec = CTF::create(buff);

Detectors/TOF/reconstruction/src/CTFCoder.cxx

@@ -218,7 +218,7 @@ size_t CTFCoder::estimateCompressedSize(const CompressedInfos& cc)
   sz += ESTSIZE(cc.tot,          CTF::BLCtot);
   sz += ESTSIZE(cc.pattMap,      CTF::BLCpattMap);
   // clang-format on
-
+  sz *= 2. / 3; // if needed, will be autoexpanded
   LOG(INFO) << "Estimated output size is " << sz << " bytes";
   return sz;
 }

Detectors/TPC/reconstruction/src/CTFCoder.cxx

@@ -215,7 +215,7 @@ size_t CTFCoder::estimateCompressedSize(const CompressedClusters& ccl)
   sz += ESTSIZE(CTF::BLCnTrackClusters,    ccl.nTrackClusters,    ccl.nTracks);
   sz += ESTSIZE(CTF::BLCnSliceRowClusters, ccl.nSliceRowClusters, ccl.nSliceRows);
   // clang-format on
-
+  sz *= 2. / 3; // if needed, will be autoexpanded
   LOG(INFO) << "Estimated output size is " << sz << " bytes";
   return sz;
 }