Skip to content

Commit 0c12fd5

Browse files
jblomerjblomer
committed
[ntuple] test automatic schema evolution of collections
1 parent 5f6bb40 commit 0c12fd5

File tree

4 files changed

+354
-0
lines changed

4 files changed

+354
-0
lines changed

‎tree/ntuple/test/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ ROOT_GENERATE_DICTIONARY(RNTupleDescriptorDict ${CMAKE_CURRENT_SOURCE_DIR}/RNTup
3838

3939
ROOT_ADD_GTEST(ntuple_endian ntuple_endian.cxx LIBRARIES ROOTNTuple)
4040
ROOT_ADD_GTEST(ntuple_evolution_type ntuple_evolution_type.cxx LIBRARIES ROOTNTuple)
41+
ROOT_GENERATE_DICTIONARY(STLContainerEvolutionDict ${CMAKE_CURRENT_SOURCE_DIR}/STLContainerEvolution.hxx
42+
MODULE ntuple_evolution_type
43+
LINKDEF STLContainerEvolutionLinkDef.h
44+
OPTIONS -inlineInputHeader
45+
DEPENDENCIES CustomStruct)
4146
if(NOT MSVC)
4247
# These unit tests rely on fork(), which is not available on Windows.
4348
ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple)

‎tree/ntuple/test/STLContainerEvolution.hxx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef ROOT_RNTuple_Test_STLContainerEvolution
2+
#define ROOT_RNTuple_Test_STLContainerEvolution
3+
4+
#include <map>
5+
#include <set>
6+
#include <unordered_set>
7+
#include <unordered_map>
8+
#include <utility>
9+
#include <vector>
10+
11+
template <typename T>
12+
struct CollectionProxy {
13+
using ValueType = T;
14+
std::vector<T> v; //!
15+
};
16+
17+
#endif

‎tree/ntuple/test/STLContainerEvolutionLinkDef.h‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#ifdef __CLING__
2+
3+
#pragma link C++ class std::set<int>+;
4+
#pragma link C++ class std::set<short int>+;
5+
#pragma link C++ class std::set<std::pair<int, int>>+;
6+
#pragma link C++ class std::set<std::pair<short int, short int>>+;
7+
8+
#pragma link C++ class std::unordered_set<int>+;
9+
#pragma link C++ class std::unordered_set<short int>+;
10+
11+
#pragma link C++ class std::multiset<int>+;
12+
#pragma link C++ class std::multiset<short int>+;
13+
#pragma link C++ class std::multiset<std::pair<int, int>>+;
14+
#pragma link C++ class std::multiset<std::pair<short int, short int>>+;
15+
16+
#pragma link C++ class std::unordered_multiset<int>+;
17+
#pragma link C++ class std::unordered_multiset<short int>+;
18+
19+
#pragma link C++ class std::map<int, int>+;
20+
#pragma link C++ class std::map<short int, short int>+;
21+
22+
#pragma link C++ class std::unordered_map<int, int>+;
23+
#pragma link C++ class std::unordered_map<short int, short int>+;
24+
25+
#pragma link C++ class std::multimap<int, int>+;
26+
#pragma link C++ class std::multimap<short int, short int>+;
27+
28+
#pragma link C++ class std::unordered_multimap<int, int>+;
29+
#pragma link C++ class std::unordered_multimap<short int, short int>+;
30+
31+
#pragma link C++ class CollectionProxy<int>+;
32+
#pragma link C++ class CollectionProxy<short int>+;
33+
#pragma link C++ class CollectionProxy<std::pair<int, int>>+;
34+
#pragma link C++ class CollectionProxy<std::pair<short int, short int>>+;
35+
36+
#endif

‎tree/ntuple/test/ntuple_evolution_type.cxx‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#include "ntuple_test.hxx"
2+
#include "SimpleCollectionProxy.hxx"
3+
#include "STLContainerEvolution.hxx"
24

35
#include <memory>
46
#include <new>
@@ -251,3 +253,297 @@ TEST(RNTupleEvolution, ArrayAsRVec)
251253
EXPECT_EQ(1, a(0)[0]);
252254
EXPECT_EQ(2, a(0)[1]);
253255
}
256+
257+
TEST(RNTupleEvolution, NullableToVector)
258+
{
259+
FileRaii fileGuard("test_ntuple_evolution_nullable_to_vector.root");
260+
{
261+
auto model = ROOT::RNTupleModel::Create();
262+
auto o = model->MakeField<std::optional<int>>("o");
263+
auto u = model->MakeField<std::unique_ptr<int>>("u");
264+
auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
265+
266+
*o = 137;
267+
*u = std::make_unique<int>(42);
268+
writer->Fill();
269+
o->reset();
270+
u->reset();
271+
writer->Fill();
272+
}
273+
274+
auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
275+
auto v1 = reader->GetView<std::vector<short int>>("o");
276+
auto v2 = reader->GetView<ROOT::RVec<short int>>("o");
277+
auto v3 = reader->GetView<std::vector<short int>>("u");
278+
auto v4 = reader->GetView<ROOT::RVec<short int>>("u");
279+
EXPECT_EQ(137, v1(0)[0]);
280+
EXPECT_EQ(137, v2(0)[0]);
281+
EXPECT_EQ(42, v3(0)[0]);
282+
EXPECT_EQ(42, v4(0)[0]);
283+
EXPECT_TRUE(v1(1).empty());
284+
EXPECT_TRUE(v2(1).empty());
285+
EXPECT_TRUE(v3(1).empty());
286+
EXPECT_TRUE(v4(1).empty());
287+
}
288+
289+
namespace {
290+
template <typename CollectionT, bool OfPairsT>
291+
void WriteCollection(std::string_view ntplName, TFile &f)
292+
{
293+
auto model = RNTupleModel::Create();
294+
auto ptrCollection = model->MakeField<CollectionT>("f");
295+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), ntplName, f);
296+
if constexpr (OfPairsT) {
297+
*ptrCollection = {{1, 2}, {3, 4}, {5, 6}};
298+
} else {
299+
*ptrCollection = {1, 2, 3};
300+
}
301+
writer->Fill();
302+
ptrCollection->clear();
303+
writer->Fill();
304+
if constexpr (OfPairsT) {
305+
*ptrCollection = {{7, 8}};
306+
} else {
307+
*ptrCollection = {4};
308+
}
309+
writer->Fill();
310+
}
311+
312+
template <typename CollectionT, bool OfPairsT>
313+
void ReadCollection(std::string_view ntplName, std::string_view path)
314+
{
315+
auto reader = RNTupleReader::Open(ntplName, path);
316+
ASSERT_EQ(3u, reader->GetNEntries());
317+
318+
auto view = reader->GetView<CollectionT>("f");
319+
CollectionT exp0;
320+
CollectionT exp2;
321+
if constexpr (OfPairsT) {
322+
exp0 = {{1, 2}, {3, 4}, {5, 6}};
323+
exp2 = {{7, 8}};
324+
} else {
325+
exp0 = {1, 2, 3};
326+
exp2 = {4};
327+
}
328+
EXPECT_EQ(exp0.size(), view(0).size());
329+
for (const auto &elem : exp0) {
330+
EXPECT_TRUE(std::find(view(0).begin(), view(0).end(), elem) != view(0).end());
331+
}
332+
EXPECT_TRUE(view(1).empty());
333+
EXPECT_EQ(exp2.size(), view(2).size());
334+
for (std::size_t i = 0; i < exp2.size(); ++i)
335+
EXPECT_EQ(*exp2.begin(), *view(2).begin());
336+
}
337+
338+
template <typename CollectionT, bool OfPairsT>
339+
void ReadCollectionFail(std::string_view ntplName, std::string_view path)
340+
{
341+
auto reader = RNTupleReader::Open(ntplName, path);
342+
ASSERT_EQ(3u, reader->GetNEntries());
343+
344+
try {
345+
reader->GetView<CollectionT>("f");
346+
FAIL() << "this case of automatic collection schema evolution should have failed";
347+
} catch (const ROOT::RException &err) {
348+
EXPECT_THAT(err.what(), testing::HasSubstr("incompatible type"));
349+
}
350+
}
351+
} // anonymous namespace
352+
353+
namespace ROOT {
354+
template <>
355+
struct IsCollectionProxy<CollectionProxy<int>> : std::true_type {};
356+
template <>
357+
struct IsCollectionProxy<CollectionProxy<short int>> : std::true_type {};
358+
template <>
359+
struct IsCollectionProxy<CollectionProxy<std::pair<int, int>>> : std::true_type {};
360+
template <>
361+
struct IsCollectionProxy<CollectionProxy<std::pair<short int, short int>>> : std::true_type {};
362+
} // namespace ROOT
363+
364+
TEST(RNTupleEvolution, Collections)
365+
{
366+
FileRaii fileGuard("test_ntuple_evolution_collections.root");
367+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "UPDATE"));
368+
369+
TClass::GetClass("CollectionProxy<int>")->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<int>>());
370+
TClass::GetClass("CollectionProxy<short int>")
371+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<short int>>());
372+
TClass::GetClass("CollectionProxy<std::pair<int, int>>")
373+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<std::pair<int, int>>>());
374+
TClass::GetClass("CollectionProxy<std::pair<short int, short int>>")
375+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<std::pair<short int, short int>>>());
376+
377+
{
378+
auto model = RNTupleModel::Create();
379+
model->AddField(ROOT::RVectorField::CreateUntyped("f", std::make_unique<RField<int>>("x")));
380+
model->Freeze();
381+
auto v = std::static_pointer_cast<std::vector<int>>(model->GetDefaultEntry().GetPtr<void>("f"));
382+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), "untyped", *f);
383+
*v = {1, 2, 3};
384+
writer->Fill();
385+
v->clear();
386+
writer->Fill();
387+
*v = {4};
388+
writer->Fill();
389+
}
390+
{
391+
auto model = RNTupleModel::Create();
392+
model->AddField(ROOT::RVectorField::CreateUntyped("f", std::make_unique<RField<std::pair<int, int>>>("x")));
393+
model->Freeze();
394+
auto v = std::static_pointer_cast<std::vector<std::pair<int, int>>>(model->GetDefaultEntry().GetPtr<void>("f"));
395+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), "untypedOfPairs", *f);
396+
*v = {{1, 2}, {3, 4}, {5, 6}};
397+
writer->Fill();
398+
v->clear();
399+
writer->Fill();
400+
*v = {{7, 8}};
401+
writer->Fill();
402+
}
403+
{
404+
auto model = RNTupleModel::Create();
405+
auto proxy = model->MakeField<CollectionProxy<int>>("f");
406+
auto writer = RNTupleWriter::Append(std::move(model), "proxy", *f);
407+
proxy->v = {1, 2, 3};
408+
writer->Fill();
409+
proxy->v.clear();
410+
writer->Fill();
411+
proxy->v = {4};
412+
writer->Fill();
413+
}
414+
{
415+
auto model = RNTupleModel::Create();
416+
auto proxy = model->MakeField<CollectionProxy<std::pair<int, int>>>("f");
417+
auto writer = RNTupleWriter::Append(std::move(model), "proxyOfPairs", *f);
418+
proxy->v = {{1, 2}, {3, 4}, {5, 6}};
419+
writer->Fill();
420+
proxy->v.clear();
421+
writer->Fill();
422+
proxy->v = {{7, 8}};
423+
writer->Fill();
424+
}
425+
426+
WriteCollection<std::vector<int>, false>("vector", *f);
427+
WriteCollection<ROOT::RVec<int>, false>("rvec", *f);
428+
WriteCollection<std::set<int>, false>("set", *f);
429+
WriteCollection<std::unordered_set<int>, false>("unordered_set", *f);
430+
WriteCollection<std::multiset<int>, false>("multiset", *f);
431+
WriteCollection<std::unordered_multiset<int>, false>("unordered_multiset", *f);
432+
WriteCollection<std::map<int, int>, true>("map", *f);
433+
WriteCollection<std::unordered_map<int, int>, true>("unordered_map", *f);
434+
WriteCollection<std::multimap<int, int>, true>("multimap", *f);
435+
WriteCollection<std::unordered_multimap<int, int>, true>("unordered_multimap", *f);
436+
437+
WriteCollection<std::vector<std::pair<int, int>>, true>("vectorOfPairs", *f);
438+
WriteCollection<ROOT::RVec<std::pair<int, int>>, true>("rvecOfPairs", *f);
439+
WriteCollection<std::set<std::pair<int, int>>, true>("setOfPairs", *f);
440+
WriteCollection<std::multiset<std::pair<int, int>>, true>("multisetOfPairs", *f);
441+
442+
// All variations written out. Now test the collection matrix.
443+
444+
ReadCollection<std::vector<short int>, false>("untyped", fileGuard.GetPath());
445+
ReadCollection<std::vector<short int>, false>("proxy", fileGuard.GetPath());
446+
ReadCollection<std::vector<short int>, false>("rvec", fileGuard.GetPath());
447+
ReadCollection<std::vector<short int>, false>("set", fileGuard.GetPath());
448+
ReadCollection<std::vector<short int>, false>("unordered_set", fileGuard.GetPath());
449+
ReadCollection<std::vector<short int>, false>("multiset", fileGuard.GetPath());
450+
ReadCollection<std::vector<short int>, false>("unordered_multiset", fileGuard.GetPath());
451+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
452+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
453+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
454+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
455+
456+
ReadCollection<ROOT::RVec<short int>, false>("untyped", fileGuard.GetPath());
457+
ReadCollection<ROOT::RVec<short int>, false>("proxy", fileGuard.GetPath());
458+
ReadCollection<ROOT::RVec<short int>, false>("vector", fileGuard.GetPath());
459+
ReadCollection<ROOT::RVec<short int>, false>("set", fileGuard.GetPath());
460+
ReadCollection<ROOT::RVec<short int>, false>("unordered_set", fileGuard.GetPath());
461+
ReadCollection<ROOT::RVec<short int>, false>("multiset", fileGuard.GetPath());
462+
ReadCollection<ROOT::RVec<short int>, false>("unordered_multiset", fileGuard.GetPath());
463+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
464+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
465+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
466+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
467+
468+
ReadCollectionFail<std::set<short int>, false>("untyped", fileGuard.GetPath());
469+
ReadCollectionFail<std::set<short int>, false>("proxy", fileGuard.GetPath());
470+
ReadCollectionFail<std::set<short int>, false>("vector", fileGuard.GetPath());
471+
ReadCollectionFail<std::set<short int>, false>("rvec", fileGuard.GetPath());
472+
ReadCollection<std::set<short int>, false>("unordered_set", fileGuard.GetPath());
473+
ReadCollectionFail<std::set<short int>, false>("multiset", fileGuard.GetPath());
474+
ReadCollectionFail<std::set<short int>, false>("unordered_multiset", fileGuard.GetPath());
475+
ReadCollection<std::set<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
476+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
477+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
478+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
479+
480+
ReadCollectionFail<std::unordered_set<short int>, false>("untyped", fileGuard.GetPath());
481+
ReadCollectionFail<std::unordered_set<short int>, false>("proxy", fileGuard.GetPath());
482+
ReadCollectionFail<std::unordered_set<short int>, false>("vector", fileGuard.GetPath());
483+
ReadCollectionFail<std::unordered_set<short int>, false>("rvec", fileGuard.GetPath());
484+
ReadCollection<std::unordered_set<short int>, false>("set", fileGuard.GetPath());
485+
ReadCollectionFail<std::unordered_set<short int>, false>("multiset", fileGuard.GetPath());
486+
ReadCollectionFail<std::unordered_set<short int>, false>("unordered_multiset", fileGuard.GetPath());
487+
488+
ReadCollection<std::multiset<short int>, false>("untyped", fileGuard.GetPath());
489+
ReadCollection<std::multiset<short int>, false>("proxy", fileGuard.GetPath());
490+
ReadCollection<std::multiset<short int>, false>("vector", fileGuard.GetPath());
491+
ReadCollection<std::multiset<short int>, false>("rvec", fileGuard.GetPath());
492+
ReadCollection<std::multiset<short int>, false>("unordered_set", fileGuard.GetPath());
493+
ReadCollection<std::multiset<short int>, false>("set", fileGuard.GetPath());
494+
ReadCollection<std::multiset<short int>, false>("unordered_multiset", fileGuard.GetPath());
495+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
496+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
497+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
498+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
499+
500+
ReadCollection<std::unordered_multiset<short int>, false>("untyped", fileGuard.GetPath());
501+
ReadCollection<std::unordered_multiset<short int>, false>("proxy", fileGuard.GetPath());
502+
ReadCollection<std::unordered_multiset<short int>, false>("vector", fileGuard.GetPath());
503+
ReadCollection<std::unordered_multiset<short int>, false>("rvec", fileGuard.GetPath());
504+
ReadCollection<std::unordered_multiset<short int>, false>("unordered_set", fileGuard.GetPath());
505+
ReadCollection<std::unordered_multiset<short int>, false>("set", fileGuard.GetPath());
506+
ReadCollection<std::unordered_multiset<short int>, false>("multiset", fileGuard.GetPath());
507+
508+
ReadCollectionFail<std::map<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
509+
ReadCollectionFail<std::map<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
510+
ReadCollectionFail<std::map<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
511+
ReadCollectionFail<std::map<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
512+
ReadCollection<std::map<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
513+
ReadCollectionFail<std::map<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
514+
ReadCollection<std::map<short int, short int>, true>("unordered_map", fileGuard.GetPath());
515+
ReadCollectionFail<std::map<short int, short int>, true>("multimap", fileGuard.GetPath());
516+
ReadCollectionFail<std::map<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
517+
518+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
519+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
520+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
521+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
522+
ReadCollection<std::unordered_map<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
523+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
524+
ReadCollection<std::unordered_map<short int, short int>, true>("map", fileGuard.GetPath());
525+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("multimap", fileGuard.GetPath());
526+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
527+
528+
ReadCollection<std::multimap<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
529+
ReadCollection<std::multimap<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
530+
ReadCollection<std::multimap<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
531+
ReadCollection<std::multimap<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
532+
ReadCollection<std::multimap<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
533+
ReadCollection<std::multimap<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
534+
ReadCollection<std::multimap<short int, short int>, true>("map", fileGuard.GetPath());
535+
ReadCollection<std::multimap<short int, short int>, true>("unordered_map", fileGuard.GetPath());
536+
ReadCollection<std::multimap<short int, short int>, true>("multimap", fileGuard.GetPath());
537+
ReadCollection<std::multimap<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
538+
539+
ReadCollection<std::unordered_multimap<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
540+
ReadCollection<std::unordered_multimap<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
541+
ReadCollection<std::unordered_multimap<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
542+
ReadCollection<std::unordered_multimap<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
543+
ReadCollection<std::unordered_multimap<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
544+
ReadCollection<std::unordered_multimap<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
545+
ReadCollection<std::unordered_multimap<short int, short int>, true>("map", fileGuard.GetPath());
546+
ReadCollection<std::unordered_multimap<short int, short int>, true>("unordered_map", fileGuard.GetPath());
547+
ReadCollection<std::unordered_multimap<short int, short int>, true>("multimap", fileGuard.GetPath());
548+
ReadCollection<std::unordered_multimap<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
549+
}

0 commit comments

Comments
 (0)