[ntuple] finalize automatic schema evolution between collections

tree/dataframe/test/datasource_ntuple.cxx

@@ -455,7 +455,7 @@ TEST_F(RNTupleDSTest, AlternativeColumnTypes)
 
    auto multipleAlternativeTypes =
       df.Define("nJets", [](const std::vector<float> &jets) { return jets.size(); }, {"jets"})
-         .Define("smallestJet", [](const std::set<float> &jets) { return *(jets.begin()); }, {"jets"})
+         .Define("smallestJet", [](const std::multiset<float> &jets) { return *(jets.begin()); }, {"jets"})
          .Min<float>("smallestJet")
          .GetValue();
    EXPECT_FLOAT_EQ(1.f, multipleAlternativeTypes);

tree/ntuple/inc/ROOT/RField/RFieldProxiedCollection.hxx

@@ -154,7 +154,7 @@ protected:
    /// (Attach() and setting fItemSize)
    RProxiedCollectionField(std::string_view fieldName, TClass *classp);
 
-   std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const final;
+   std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const override;
    const RColumnRepresentations &GetColumnRepresentations() const final;
    void GenerateColumns() final;
    void GenerateColumns(const ROOT::RNTupleDescriptor &desc) final;
@@ -165,7 +165,7 @@ protected:
    std::size_t AppendImpl(const void *from) final;
    void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final;
 
-   void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
+   void ReconcileOnDiskField(const RNTupleDescriptor &desc) override;
 
    void CommitClusterImpl() final { fNWritten = 0; }
 
@@ -277,6 +277,15 @@ public:
       kMultiMap,
       kUnorderedMultiMap
    };
+
+private:
+   EMapType fMapType;
+
+protected:
+   std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const final;
+   void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
+
+public:
    RMapField(std::string_view fieldName, EMapType mapType, std::unique_ptr<RFieldBase> itemField);
    RMapField(RMapField &&other) = default;
    RMapField &operator=(RMapField &&other) = default;
@@ -364,6 +373,15 @@ public:
       kMultiSet,
       kUnorderedMultiSet
    };
+
+private:
+   ESetType fSetType;
+
+protected:
+   std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const final;
+   void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
+
+public:
    RSetField(std::string_view fieldName, ESetType setType, std::unique_ptr<RFieldBase> itemField);
    RSetField(RSetField &&other) = default;
    RSetField &operator=(RSetField &&other) = default;

tree/ntuple/src/RFieldMeta.cxx

@@ -880,23 +880,66 @@ void ROOT::RProxiedCollectionField::AcceptVisitor(ROOT::Detail::RFieldVisitor &v
 //------------------------------------------------------------------------------
 
 ROOT::RMapField::RMapField(std::string_view fieldName, EMapType mapType, std::unique_ptr<RFieldBase> itemField)
-   : RProxiedCollectionField(fieldName, EnsureValidClass(BuildMapTypeName(mapType, itemField.get())))
+   : RProxiedCollectionField(fieldName, EnsureValidClass(BuildMapTypeName(mapType, itemField.get()))), fMapType(mapType)
 {
    auto *itemClass = fProxy->GetValueClass();
    fItemSize = itemClass->GetClassSize();
 
    Attach(std::move(itemField));
 }
 
+std::unique_ptr<ROOT::RFieldBase> ROOT::RMapField::CloneImpl(std::string_view newName) const
+{
+   return std::make_unique<RMapField>(newName, fMapType, fSubfields[0]->Clone(fSubfields[0]->GetFieldName()));
+}
+
+void ROOT::RMapField::ReconcileOnDiskField(const RNTupleDescriptor &desc)
+{
+   static const std::vector<std::string> prefixesRegular = {"std::map<", "std::unordered_map<", "std::set<",
+                                                            "std::unordered_set<"};
+
+   EnsureMatchingOnDiskField(desc, kDiffTypeName).ThrowOnError();
+
+   switch (fMapType) {
+   case EMapType::kMap:
+   case EMapType::kUnorderedMap: EnsureMatchingTypePrefix(desc, prefixesRegular).ThrowOnError(); break;
+   default:
+      break;
+      // no restrictions for multimaps
+   }
+}
+
 //------------------------------------------------------------------------------
 
 ROOT::RSetField::RSetField(std::string_view fieldName, ESetType setType, std::unique_ptr<RFieldBase> itemField)
-   : ROOT::RProxiedCollectionField(fieldName, EnsureValidClass(BuildSetTypeName(setType, *itemField)))
+   : ROOT::RProxiedCollectionField(fieldName, EnsureValidClass(BuildSetTypeName(setType, *itemField))),
+     fSetType(setType)
 {
    fItemSize = itemField->GetValueSize();
    Attach(std::move(itemField));
 }
 
+std::unique_ptr<ROOT::RFieldBase> ROOT::RSetField::CloneImpl(std::string_view newName) const
+{
+   return std::make_unique<RSetField>(newName, fSetType, fSubfields[0]->Clone(fSubfields[0]->GetFieldName()));
+}
+
+void ROOT::RSetField::ReconcileOnDiskField(const RNTupleDescriptor &desc)
+{
+   static const std::vector<std::string> prefixesRegular = {"std::set<", "std::unordered_set<", "std::map<",
+                                                            "std::unordered_map<"};
+
+   EnsureMatchingOnDiskField(desc, kDiffTypeName).ThrowOnError();
+
+   switch (fSetType) {
+   case ESetType::kSet:
+   case ESetType::kUnorderedSet: EnsureMatchingTypePrefix(desc, prefixesRegular).ThrowOnError(); break;
+   default:
+      break;
+      // no restrictions for multisets
+   }
+}
+
 //------------------------------------------------------------------------------
 
 namespace {

tree/ntuple/src/RFieldSequenceContainer.cxx

@@ -547,7 +547,7 @@ ROOT::RVectorField::RVectorField(std::string_view fieldName, std::unique_ptr<RFi
 std::unique_ptr<ROOT::RVectorField>
 ROOT::RVectorField::CreateUntyped(std::string_view fieldName, std::unique_ptr<RFieldBase> itemField)
 {
-   return std::unique_ptr<ROOT::RVectorField>(new RVectorField(fieldName, std::move(itemField), ""));
+   return std::unique_ptr<ROOT::RVectorField>(new RVectorField(fieldName, itemField->Clone("_0"), ""));
 }
 
 std::unique_ptr<ROOT::RFieldBase> ROOT::RVectorField::CloneImpl(std::string_view newName) const

tree/ntuple/test/CMakeLists.txt

@@ -38,6 +38,11 @@ ROOT_GENERATE_DICTIONARY(RNTupleDescriptorDict ${CMAKE_CURRENT_SOURCE_DIR}/RNTup
 
 ROOT_ADD_GTEST(ntuple_endian ntuple_endian.cxx LIBRARIES ROOTNTuple)
 ROOT_ADD_GTEST(ntuple_evolution_type ntuple_evolution_type.cxx LIBRARIES ROOTNTuple)
+ROOT_GENERATE_DICTIONARY(STLContainerEvolutionDict ${CMAKE_CURRENT_SOURCE_DIR}/STLContainerEvolution.hxx
+                         MODULE ntuple_evolution_type
+                         LINKDEF STLContainerEvolutionLinkDef.h
+                         OPTIONS -inlineInputHeader
+                         DEPENDENCIES CustomStruct)
 if(NOT MSVC)
   # These unit tests rely on fork(), which is not available on Windows.
   ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple)

tree/ntuple/test/STLContainerEvolution.hxx

@@ -0,0 +1,39 @@
+#ifndef ROOT_RNTuple_Test_STLContainerEvolution
+#define ROOT_RNTuple_Test_STLContainerEvolution
+
+#include <map>
+#include <set>
+#include <unordered_set>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+// The following std::hash specializations allow for creating unordered sets of pairs in the Collections test.
+// It is a quick but bad hash but for this test though it should be ok.
+template <>
+struct std::hash<std::pair<int, int>> {
+   std::size_t operator()(const std::pair<int, int> &p) const noexcept
+   {
+      std::size_t h1 = std::hash<int>{}(p.first);
+      std::size_t h2 = std::hash<int>{}(p.second);
+      return h1 ^ (h2 << 1);
+   }
+};
+
+template <>
+struct std::hash<std::pair<short int, short int>> {
+   std::size_t operator()(const std::pair<short int, short int> &p) const noexcept
+   {
+      std::size_t h1 = std::hash<short int>{}(p.first);
+      std::size_t h2 = std::hash<short int>{}(p.second);
+      return h1 ^ (h2 << 1);
+   }
+};
+
+template <typename T>
+struct CollectionProxy {
+   using ValueType = T;
+   std::vector<T> v; //!
+};
+
+#endif

tree/ntuple/test/STLContainerEvolutionLinkDef.h

@@ -0,0 +1,40 @@
+#ifdef __CLING__
+
+#pragma link C++ class std::set<int>+;
+#pragma link C++ class std::set<short int>+;
+#pragma link C++ class std::set<std::pair<int, int>>+;
+#pragma link C++ class std::set<std::pair<short int, short int>>+;
+
+#pragma link C++ class std::unordered_set<int>+;
+#pragma link C++ class std::unordered_set<short int>+;
+#pragma link C++ class std::unordered_set<std::pair<int, int>>+;
+#pragma link C++ class std::unordered_set<std::pair<short int, short int>>+;
+
+#pragma link C++ class std::multiset<int>+;
+#pragma link C++ class std::multiset<short int>+;
+#pragma link C++ class std::multiset<std::pair<int, int>>+;
+#pragma link C++ class std::multiset<std::pair<short int, short int>>+;
+
+#pragma link C++ class std::unordered_multiset<int>+;
+#pragma link C++ class std::unordered_multiset<short int>+;
+#pragma link C++ class std::unordered_multiset<std::pair<int, int>>+;
+#pragma link C++ class std::unordered_multiset<std::pair<short int, short int>>+;
+
+#pragma link C++ class std::map<int, int>+;
+#pragma link C++ class std::map<short int, short int>+;
+
+#pragma link C++ class std::unordered_map<int, int>+;
+#pragma link C++ class std::unordered_map<short int, short int>+;
+
+#pragma link C++ class std::multimap<int, int>+;
+#pragma link C++ class std::multimap<short int, short int>+;
+
+#pragma link C++ class std::unordered_multimap<int, int>+;
+#pragma link C++ class std::unordered_multimap<short int, short int>+;
+
+#pragma link C++ class CollectionProxy<int>+;
+#pragma link C++ class CollectionProxy<short int>+;
+#pragma link C++ class CollectionProxy<std::pair<int, int>>+;
+#pragma link C++ class CollectionProxy<std::pair<short int, short int>>+;
+
+#endif