Skip to content

Commit f420cd5

Browse files
jblomerjblomer
committed
[ntuple] test automatic schema evolution of collections
1 parent 3d9bf61 commit f420cd5

File tree

4 files changed

+355
-0
lines changed

4 files changed

+355
-0
lines changed

‎tree/ntuple/test/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ ROOT_GENERATE_DICTIONARY(RNTupleDescriptorDict ${CMAKE_CURRENT_SOURCE_DIR}/RNTup
3838

3939
ROOT_ADD_GTEST(ntuple_endian ntuple_endian.cxx LIBRARIES ROOTNTuple)
4040
ROOT_ADD_GTEST(ntuple_evolution_type ntuple_evolution_type.cxx LIBRARIES ROOTNTuple)
41+
ROOT_GENERATE_DICTIONARY(STLContainerEvolutionDict ${CMAKE_CURRENT_SOURCE_DIR}/STLContainerEvolution.hxx
42+
MODULE ntuple_evolution_type
43+
LINKDEF STLContainerEvolutionLinkDef.h
44+
OPTIONS -inlineInputHeader
45+
DEPENDENCIES CustomStruct)
4146
if(NOT MSVC)
4247
# These unit tests rely on fork(), which is not available on Windows.
4348
ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple)

‎tree/ntuple/test/STLContainerEvolution.hxx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef ROOT_RNTuple_Test_STLContainerEvolution
2+
#define ROOT_RNTuple_Test_STLContainerEvolution
3+
4+
#include <map>
5+
#include <set>
6+
#include <unordered_set>
7+
#include <unordered_map>
8+
#include <utility>
9+
#include <vector>
10+
11+
template <typename T>
12+
struct CollectionProxy {
13+
using ValueType = T;
14+
std::vector<T> v; //!
15+
};
16+
17+
#endif

‎tree/ntuple/test/STLContainerEvolutionLinkDef.h‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#ifdef __CLING__
2+
3+
#pragma link C++ class std::set<int>+;
4+
#pragma link C++ class std::set<short int>+;
5+
#pragma link C++ class std::set<std::pair<int, int>>+;
6+
#pragma link C++ class std::set<std::pair<short int, short int>>+;
7+
8+
#pragma link C++ class std::unordered_set<int>+;
9+
#pragma link C++ class std::unordered_set<short int>+;
10+
11+
#pragma link C++ class std::multiset<int>+;
12+
#pragma link C++ class std::multiset<short int>+;
13+
#pragma link C++ class std::multiset<std::pair<int, int>>+;
14+
#pragma link C++ class std::multiset<std::pair<short int, short int>>+;
15+
16+
#pragma link C++ class std::unordered_multiset<int>+;
17+
#pragma link C++ class std::unordered_multiset<short int>+;
18+
19+
#pragma link C++ class std::map<int, int>+;
20+
#pragma link C++ class std::map<short int, short int>+;
21+
22+
#pragma link C++ class std::unordered_map<int, int>+;
23+
#pragma link C++ class std::unordered_map<short int, short int>+;
24+
25+
#pragma link C++ class std::multimap<int, int>+;
26+
#pragma link C++ class std::multimap<short int, short int>+;
27+
28+
#pragma link C++ class std::unordered_multimap<int, int>+;
29+
#pragma link C++ class std::unordered_multimap<short int, short int>+;
30+
31+
#pragma link C++ class CollectionProxy<int>+;
32+
#pragma link C++ class CollectionProxy<short int>+;
33+
#pragma link C++ class CollectionProxy<std::pair<int, int>>+;
34+
#pragma link C++ class CollectionProxy<std::pair<short int, short int>>+;
35+
36+
#endif

‎tree/ntuple/test/ntuple_evolution_type.cxx‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#include "ntuple_test.hxx"
2+
#include "SimpleCollectionProxy.hxx"
3+
#include "STLContainerEvolution.hxx"
24

35
#include <memory>
46
#include <new>
@@ -251,3 +253,298 @@ TEST(RNTupleEvolution, ArrayAsRVec)
251253
EXPECT_EQ(1, a(0)[0]);
252254
EXPECT_EQ(2, a(0)[1]);
253255
}
256+
257+
TEST(RNTupleEvolution, NullableToVector)
258+
{
259+
FileRaii fileGuard("test_ntuple_evolution_nullable_to_vector.root");
260+
{
261+
auto model = ROOT::RNTupleModel::Create();
262+
auto o = model->MakeField<std::optional<int>>("o");
263+
auto u = model->MakeField<std::unique_ptr<int>>("u");
264+
auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
265+
266+
*o = 137;
267+
*u = std::make_unique<int>(42);
268+
writer->Fill();
269+
o->reset();
270+
u->reset();
271+
writer->Fill();
272+
}
273+
274+
auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
275+
auto v1 = reader->GetView<std::vector<short int>>("o");
276+
auto v2 = reader->GetView<ROOT::RVec<short int>>("o");
277+
auto v3 = reader->GetView<std::vector<short int>>("u");
278+
auto v4 = reader->GetView<ROOT::RVec<short int>>("u");
279+
EXPECT_EQ(137, v1(0)[0]);
280+
EXPECT_EQ(137, v2(0)[0]);
281+
EXPECT_EQ(42, v3(0)[0]);
282+
EXPECT_EQ(42, v4(0)[0]);
283+
EXPECT_TRUE(v1(1).empty());
284+
EXPECT_TRUE(v2(1).empty());
285+
EXPECT_TRUE(v3(1).empty());
286+
EXPECT_TRUE(v4(1).empty());
287+
}
288+
289+
namespace {
290+
template <typename CollectionT, bool OfPairsT>
291+
void WriteCollection(std::string_view ntplName, TFile &f)
292+
{
293+
auto model = RNTupleModel::Create();
294+
auto ptrCollection = model->MakeField<CollectionT>("f");
295+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), ntplName, f);
296+
if constexpr (OfPairsT) {
297+
*ptrCollection = {{1, 2}, {3, 4}, {5, 6}};
298+
} else {
299+
*ptrCollection = {1, 2, 3};
300+
}
301+
writer->Fill();
302+
ptrCollection->clear();
303+
writer->Fill();
304+
if constexpr (OfPairsT) {
305+
*ptrCollection = {{7, 8}};
306+
} else {
307+
*ptrCollection = {4};
308+
}
309+
writer->Fill();
310+
}
311+
312+
template <typename CollectionT, bool OfPairsT>
313+
void ReadCollection(std::string_view ntplName, std::string_view path)
314+
{
315+
auto reader = RNTupleReader::Open(ntplName, path);
316+
ASSERT_EQ(3u, reader->GetNEntries());
317+
318+
auto view = reader->GetView<CollectionT>("f");
319+
CollectionT exp0;
320+
CollectionT exp2;
321+
if constexpr (OfPairsT) {
322+
exp0 = {{1, 2}, {3, 4}, {5, 6}};
323+
exp2 = {{7, 8}};
324+
} else {
325+
exp0 = {1, 2, 3};
326+
exp2 = {4};
327+
}
328+
EXPECT_EQ(exp0.size(), view(0).size());
329+
for (const auto &elem : exp0) {
330+
const auto &ref = view(0);
331+
EXPECT_TRUE(std::find(ref.begin(), ref.end(), elem) != ref.end());
332+
}
333+
EXPECT_TRUE(view(1).empty());
334+
EXPECT_EQ(exp2.size(), view(2).size());
335+
for (std::size_t i = 0; i < exp2.size(); ++i)
336+
EXPECT_EQ(*exp2.begin(), *view(2).begin());
337+
}
338+
339+
template <typename CollectionT, bool OfPairsT>
340+
void ReadCollectionFail(std::string_view ntplName, std::string_view path)
341+
{
342+
auto reader = RNTupleReader::Open(ntplName, path);
343+
ASSERT_EQ(3u, reader->GetNEntries());
344+
345+
try {
346+
reader->GetView<CollectionT>("f");
347+
FAIL() << "this case of automatic collection schema evolution should have failed";
348+
} catch (const ROOT::RException &err) {
349+
EXPECT_THAT(err.what(), testing::HasSubstr("incompatible type"));
350+
}
351+
}
352+
} // anonymous namespace
353+
354+
namespace ROOT {
355+
template <>
356+
struct IsCollectionProxy<CollectionProxy<int>> : std::true_type {};
357+
template <>
358+
struct IsCollectionProxy<CollectionProxy<short int>> : std::true_type {};
359+
template <>
360+
struct IsCollectionProxy<CollectionProxy<std::pair<int, int>>> : std::true_type {};
361+
template <>
362+
struct IsCollectionProxy<CollectionProxy<std::pair<short int, short int>>> : std::true_type {};
363+
} // namespace ROOT
364+
365+
TEST(RNTupleEvolution, Collections)
366+
{
367+
FileRaii fileGuard("test_ntuple_evolution_collections.root");
368+
auto f = std::unique_ptr<TFile>(TFile::Open(fileGuard.GetPath().c_str(), "UPDATE"));
369+
370+
TClass::GetClass("CollectionProxy<int>")->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<int>>());
371+
TClass::GetClass("CollectionProxy<short int>")
372+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<short int>>());
373+
TClass::GetClass("CollectionProxy<std::pair<int, int>>")
374+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<std::pair<int, int>>>());
375+
TClass::GetClass("CollectionProxy<std::pair<short int, short int>>")
376+
->CopyCollectionProxy(SimpleCollectionProxy<CollectionProxy<std::pair<short int, short int>>>());
377+
378+
{
379+
auto model = RNTupleModel::Create();
380+
model->AddField(ROOT::RVectorField::CreateUntyped("f", std::make_unique<RField<int>>("x")));
381+
model->Freeze();
382+
auto v = std::static_pointer_cast<std::vector<int>>(model->GetDefaultEntry().GetPtr<void>("f"));
383+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), "untyped", *f);
384+
*v = {1, 2, 3};
385+
writer->Fill();
386+
v->clear();
387+
writer->Fill();
388+
*v = {4};
389+
writer->Fill();
390+
}
391+
{
392+
auto model = RNTupleModel::Create();
393+
model->AddField(ROOT::RVectorField::CreateUntyped("f", std::make_unique<RField<std::pair<int, int>>>("x")));
394+
model->Freeze();
395+
auto v = std::static_pointer_cast<std::vector<std::pair<int, int>>>(model->GetDefaultEntry().GetPtr<void>("f"));
396+
auto writer = ROOT::RNTupleWriter::Append(std::move(model), "untypedOfPairs", *f);
397+
*v = {{1, 2}, {3, 4}, {5, 6}};
398+
writer->Fill();
399+
v->clear();
400+
writer->Fill();
401+
*v = {{7, 8}};
402+
writer->Fill();
403+
}
404+
{
405+
auto model = RNTupleModel::Create();
406+
auto proxy = model->MakeField<CollectionProxy<int>>("f");
407+
auto writer = RNTupleWriter::Append(std::move(model), "proxy", *f);
408+
proxy->v = {1, 2, 3};
409+
writer->Fill();
410+
proxy->v.clear();
411+
writer->Fill();
412+
proxy->v = {4};
413+
writer->Fill();
414+
}
415+
{
416+
auto model = RNTupleModel::Create();
417+
auto proxy = model->MakeField<CollectionProxy<std::pair<int, int>>>("f");
418+
auto writer = RNTupleWriter::Append(std::move(model), "proxyOfPairs", *f);
419+
proxy->v = {{1, 2}, {3, 4}, {5, 6}};
420+
writer->Fill();
421+
proxy->v.clear();
422+
writer->Fill();
423+
proxy->v = {{7, 8}};
424+
writer->Fill();
425+
}
426+
427+
WriteCollection<std::vector<int>, false>("vector", *f);
428+
WriteCollection<ROOT::RVec<int>, false>("rvec", *f);
429+
WriteCollection<std::set<int>, false>("set", *f);
430+
WriteCollection<std::unordered_set<int>, false>("unordered_set", *f);
431+
WriteCollection<std::multiset<int>, false>("multiset", *f);
432+
WriteCollection<std::unordered_multiset<int>, false>("unordered_multiset", *f);
433+
WriteCollection<std::map<int, int>, true>("map", *f);
434+
WriteCollection<std::unordered_map<int, int>, true>("unordered_map", *f);
435+
WriteCollection<std::multimap<int, int>, true>("multimap", *f);
436+
WriteCollection<std::unordered_multimap<int, int>, true>("unordered_multimap", *f);
437+
438+
WriteCollection<std::vector<std::pair<int, int>>, true>("vectorOfPairs", *f);
439+
WriteCollection<ROOT::RVec<std::pair<int, int>>, true>("rvecOfPairs", *f);
440+
WriteCollection<std::set<std::pair<int, int>>, true>("setOfPairs", *f);
441+
WriteCollection<std::multiset<std::pair<int, int>>, true>("multisetOfPairs", *f);
442+
443+
// All variations written out. Now test the collection matrix.
444+
445+
ReadCollection<std::vector<short int>, false>("untyped", fileGuard.GetPath());
446+
ReadCollection<std::vector<short int>, false>("proxy", fileGuard.GetPath());
447+
ReadCollection<std::vector<short int>, false>("rvec", fileGuard.GetPath());
448+
ReadCollection<std::vector<short int>, false>("set", fileGuard.GetPath());
449+
ReadCollection<std::vector<short int>, false>("unordered_set", fileGuard.GetPath());
450+
ReadCollection<std::vector<short int>, false>("multiset", fileGuard.GetPath());
451+
ReadCollection<std::vector<short int>, false>("unordered_multiset", fileGuard.GetPath());
452+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
453+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
454+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
455+
ReadCollection<std::vector<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
456+
457+
ReadCollection<ROOT::RVec<short int>, false>("untyped", fileGuard.GetPath());
458+
ReadCollection<ROOT::RVec<short int>, false>("proxy", fileGuard.GetPath());
459+
ReadCollection<ROOT::RVec<short int>, false>("vector", fileGuard.GetPath());
460+
ReadCollection<ROOT::RVec<short int>, false>("set", fileGuard.GetPath());
461+
ReadCollection<ROOT::RVec<short int>, false>("unordered_set", fileGuard.GetPath());
462+
ReadCollection<ROOT::RVec<short int>, false>("multiset", fileGuard.GetPath());
463+
ReadCollection<ROOT::RVec<short int>, false>("unordered_multiset", fileGuard.GetPath());
464+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
465+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
466+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
467+
ReadCollection<ROOT::RVec<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
468+
469+
ReadCollectionFail<std::set<short int>, false>("untyped", fileGuard.GetPath());
470+
ReadCollectionFail<std::set<short int>, false>("proxy", fileGuard.GetPath());
471+
ReadCollectionFail<std::set<short int>, false>("vector", fileGuard.GetPath());
472+
ReadCollectionFail<std::set<short int>, false>("rvec", fileGuard.GetPath());
473+
ReadCollection<std::set<short int>, false>("unordered_set", fileGuard.GetPath());
474+
ReadCollectionFail<std::set<short int>, false>("multiset", fileGuard.GetPath());
475+
ReadCollectionFail<std::set<short int>, false>("unordered_multiset", fileGuard.GetPath());
476+
ReadCollection<std::set<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
477+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
478+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
479+
ReadCollectionFail<std::set<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
480+
481+
ReadCollectionFail<std::unordered_set<short int>, false>("untyped", fileGuard.GetPath());
482+
ReadCollectionFail<std::unordered_set<short int>, false>("proxy", fileGuard.GetPath());
483+
ReadCollectionFail<std::unordered_set<short int>, false>("vector", fileGuard.GetPath());
484+
ReadCollectionFail<std::unordered_set<short int>, false>("rvec", fileGuard.GetPath());
485+
ReadCollection<std::unordered_set<short int>, false>("set", fileGuard.GetPath());
486+
ReadCollectionFail<std::unordered_set<short int>, false>("multiset", fileGuard.GetPath());
487+
ReadCollectionFail<std::unordered_set<short int>, false>("unordered_multiset", fileGuard.GetPath());
488+
489+
ReadCollection<std::multiset<short int>, false>("untyped", fileGuard.GetPath());
490+
ReadCollection<std::multiset<short int>, false>("proxy", fileGuard.GetPath());
491+
ReadCollection<std::multiset<short int>, false>("vector", fileGuard.GetPath());
492+
ReadCollection<std::multiset<short int>, false>("rvec", fileGuard.GetPath());
493+
ReadCollection<std::multiset<short int>, false>("unordered_set", fileGuard.GetPath());
494+
ReadCollection<std::multiset<short int>, false>("set", fileGuard.GetPath());
495+
ReadCollection<std::multiset<short int>, false>("unordered_multiset", fileGuard.GetPath());
496+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("map", fileGuard.GetPath());
497+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("unordered_map", fileGuard.GetPath());
498+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("multimap", fileGuard.GetPath());
499+
ReadCollection<std::multiset<std::pair<short int, short int>>, true>("unordered_multimap", fileGuard.GetPath());
500+
501+
ReadCollection<std::unordered_multiset<short int>, false>("untyped", fileGuard.GetPath());
502+
ReadCollection<std::unordered_multiset<short int>, false>("proxy", fileGuard.GetPath());
503+
ReadCollection<std::unordered_multiset<short int>, false>("vector", fileGuard.GetPath());
504+
ReadCollection<std::unordered_multiset<short int>, false>("rvec", fileGuard.GetPath());
505+
ReadCollection<std::unordered_multiset<short int>, false>("unordered_set", fileGuard.GetPath());
506+
ReadCollection<std::unordered_multiset<short int>, false>("set", fileGuard.GetPath());
507+
ReadCollection<std::unordered_multiset<short int>, false>("multiset", fileGuard.GetPath());
508+
509+
ReadCollectionFail<std::map<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
510+
ReadCollectionFail<std::map<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
511+
ReadCollectionFail<std::map<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
512+
ReadCollectionFail<std::map<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
513+
ReadCollection<std::map<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
514+
ReadCollectionFail<std::map<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
515+
ReadCollection<std::map<short int, short int>, true>("unordered_map", fileGuard.GetPath());
516+
ReadCollectionFail<std::map<short int, short int>, true>("multimap", fileGuard.GetPath());
517+
ReadCollectionFail<std::map<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
518+
519+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
520+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
521+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
522+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
523+
ReadCollection<std::unordered_map<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
524+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
525+
ReadCollection<std::unordered_map<short int, short int>, true>("map", fileGuard.GetPath());
526+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("multimap", fileGuard.GetPath());
527+
ReadCollectionFail<std::unordered_map<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
528+
529+
ReadCollection<std::multimap<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
530+
ReadCollection<std::multimap<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
531+
ReadCollection<std::multimap<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
532+
ReadCollection<std::multimap<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
533+
ReadCollection<std::multimap<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
534+
ReadCollection<std::multimap<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
535+
ReadCollection<std::multimap<short int, short int>, true>("map", fileGuard.GetPath());
536+
ReadCollection<std::multimap<short int, short int>, true>("unordered_map", fileGuard.GetPath());
537+
ReadCollection<std::multimap<short int, short int>, true>("multimap", fileGuard.GetPath());
538+
ReadCollection<std::multimap<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
539+
540+
ReadCollection<std::unordered_multimap<short int, short int>, true>("untypedOfPairs", fileGuard.GetPath());
541+
ReadCollection<std::unordered_multimap<short int, short int>, true>("proxyOfPairs", fileGuard.GetPath());
542+
ReadCollection<std::unordered_multimap<short int, short int>, true>("vectorOfPairs", fileGuard.GetPath());
543+
ReadCollection<std::unordered_multimap<short int, short int>, true>("rvecOfPairs", fileGuard.GetPath());
544+
ReadCollection<std::unordered_multimap<short int, short int>, true>("setOfPairs", fileGuard.GetPath());
545+
ReadCollection<std::unordered_multimap<short int, short int>, true>("multisetOfPairs", fileGuard.GetPath());
546+
ReadCollection<std::unordered_multimap<short int, short int>, true>("map", fileGuard.GetPath());
547+
ReadCollection<std::unordered_multimap<short int, short int>, true>("unordered_map", fileGuard.GetPath());
548+
ReadCollection<std::unordered_multimap<short int, short int>, true>("multimap", fileGuard.GetPath());
549+
ReadCollection<std::unordered_multimap<short int, short int>, true>("unordered_multimap", fileGuard.GetPath());
550+
}

0 commit comments

Comments
 (0)