serialization of matchers

Author: aalkin

Framework/Core/include/Framework/DataSpecUtils.h

@@ -214,6 +214,9 @@ struct DataSpecUtils {
   /// Create an InputSpec from metadata string
   static InputSpec fromMetadataString(std::string s);
 
+  /// Create a concrete data matcher from serialized string
+  static ConcreteDataMatcher fromString(std::string s);
+
   /// Get the origin, if available
   static std::optional<header::DataOrigin> getOptionalOrigin(InputSpec const& spec);
 

Framework/Core/include/Framework/DataSpecViews.h

@@ -43,6 +43,13 @@ static auto filter_string_params_with(std::string match)
   });
 }
 
+static auto filter_string_params_starts_with(std::string match)
+{
+  return std::views::filter([match](auto const& param) {
+    return (param.type == VariantType::String) && (param.name.starts_with(match));
+  });
+}
+
 static auto input_to_output_specs()
 {
   return std::views::transform([](auto const& input) {

Framework/Core/include/Framework/InputRecord.h

@@ -513,15 +513,22 @@ class InputRecord
   }
 
   template <typename T>
-    requires(std::same_as<T, TableConsumer>)
+    requires(std::same_as<T, DataRef>)
   decltype(auto) get(ConcreteDataMatcher matcher, int part = 0)
   {
     auto pos = getPos(matcher);
     if (pos < 0) {
       auto msg = describeAvailableInputs();
-      throw runtime_error_f("InputRecord::get: no input with binding %s found. %s", DataSpecUtils::describe(matcher), msg.c_str());
+      throw runtime_error_f("InputRecord::get: no input with binding %s found. %s", DataSpecUtils::describe(matcher).c_str(), msg.c_str());
     }
-    auto ref = getByPos(pos, part);
+    return getByPos(pos, part);
+  }
+
+  template <typename T>
+    requires(std::same_as<T, TableConsumer>)
+  decltype(auto) get(ConcreteDataMatcher matcher, int part = 0)
+  {
+    auto ref = get<DataRef>(matcher, part);
     auto data = reinterpret_cast<uint8_t const*>(ref.payload);
     return std::make_unique<TableConsumer>(data, DataRefUtils::getPayloadSize(ref));
   }

Framework/Core/src/DataSpecUtils.cxx

@@ -669,16 +669,39 @@ InputSpec DataSpecUtils::fromMetadataString(std::string s)
   if (std::distance(words, std::sregex_iterator()) != 4) {
     throw runtime_error_f("Malformed input spec metadata: %s", s.c_str());
   }
-  std::vector<std::string> data;
+  std::array<std::string, 4> data;
+  auto pos = 0;
   for (auto i = words; i != std::sregex_iterator(); ++i) {
-    data.emplace_back(i->str());
+    data[pos] = i->str();
+    ++pos;
   }
   char origin[4];
   char description[16];
   std::memcpy(&origin, data[1].c_str(), 4);
   std::memcpy(&description, data[2].c_str(), 16);
   auto version = static_cast<o2::header::DataHeader::SubSpecificationType>(std::atoi(data[3].c_str()));
-  return InputSpec{data[0], header::DataOrigin{origin}, header::DataDescription{description}, version, Lifetime::Timeframe};
+  return {data[0], header::DataOrigin{origin}, header::DataDescription{description}, version, Lifetime::Timeframe};
+}
+
+ConcreteDataMatcher DataSpecUtils::fromString(std::string s)
+{
+  std::regex word_regex("(\\w+)");
+  auto words = std::sregex_iterator(s.begin(), s.end(), word_regex);
+  if (std::distance(words, std::sregex_iterator()) != 3) {
+    throw runtime_error_f("Malformed serialized matcher: %s", s.c_str());
+  }
+  std::array<std::string, 3> data;
+  auto pos = 0;
+  for (auto i = words; i != std::sregex_iterator(); ++i) {
+    data[pos] = i->str();
+    ++pos;
+  }
+  char origin[4];
+  char description[16];
+  std::memcpy(&origin, data[0].c_str(), 4);
+  std::memcpy(&description, data[1].c_str(), 16);
+  auto version = static_cast<o2::header::DataHeader::SubSpecificationType>(std::atoi(data[2].c_str()));
+  return {header::DataOrigin{origin}, header::DataDescription{description}, version};
 }
 
 std::optional<header::DataOrigin> DataSpecUtils::getOptionalOrigin(InputSpec const& spec)

Framework/Core/src/IndexJSONHelpers.cxx

@@ -41,6 +41,7 @@ struct IndexRecordsReader : public rapidjson::BaseReaderHandler<rapidjson::UTF8<
   std::string currentKey;
   std::string label;
   std::string columnLabel;
+  std::string matcherStr;
   o2::soa::IndexKind kind;
   int pos;
 
@@ -87,6 +88,9 @@ struct IndexRecordsReader : public rapidjson::BaseReaderHandler<rapidjson::UTF8<
       if (currentKey.compare("label") == 0) {
         return true;
       }
+      if (currentKey.compare("matcher") == 0) {
+        return true;
+      }
       if (currentKey.compare("column") == 0) {
         return true;
       }
@@ -127,7 +131,7 @@ struct IndexRecordsReader : public rapidjson::BaseReaderHandler<rapidjson::UTF8<
     if (states.top() == State::IN_RECORD) {
       states.pop();
       // add a record
-      records.emplace_back(label, columnLabel, kind, pos);
+      records.emplace_back(label, DataSpecUtils::fromString(matcherStr), columnLabel, kind, pos);
       return true;
     }
 
@@ -175,6 +179,10 @@ struct IndexRecordsReader : public rapidjson::BaseReaderHandler<rapidjson::UTF8<
         columnLabel = str;
         return true;
       }
+      if (currentKey.compare("matcher") == 0) {
+        matcherStr = str;
+        return true;
+      }
     }
 
     states.push(State::IN_ERROR);
@@ -205,6 +213,8 @@ void writeRecords(rapidjson::Writer<rapidjson::OStreamWrapper>& w, std::vector<o
     w.StartObject();
     w.Key("label");
     w.String(r.label.c_str());
+    w.Key("matcher");
+    w.String(DataSpecUtils::describe(r.matcher).c_str());
     w.Key("column");
     w.String(r.columnLabel.c_str());
     w.Key("kind");