Skip to content

Commit 154c97c

Browse files
SunjeetSunjeet
committed
List joiner tests
1 parent 730d157 commit 154c97c

File tree

1 file changed

+169
-0
lines changed

1 file changed

+169
-0
lines changed

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsJoinerTest.java

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
package com.netflix.hollow.core.read.engine.object;
2+
3+
import static org.junit.Assert.assertEquals;
4+
import static org.junit.Assert.assertTrue;
5+
6+
import com.netflix.hollow.api.consumer.HollowConsumer;
7+
import com.netflix.hollow.api.producer.HollowProducer;
8+
import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
9+
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
10+
import com.netflix.hollow.test.InMemoryBlobStore;
11+
import java.io.IOException;
12+
import org.junit.Assert;
13+
import org.junit.Test;
14+
15+
public class HollowObjectTypeDataElementsJoinerTest extends AbstractHollowObjectTypeDataElementsSplitJoinTest {
16+
@Override
17+
protected void initializeTypeStates() {
18+
super.initializeTypeStates();
19+
writeStateEngine.setTargetMaxTypeShardSize(16);
20+
}
21+
22+
@Test
23+
public void testJoin() throws IOException {
24+
HollowObjectTypeReadState typeReadState = populateTypeStateWith(1);
25+
assertEquals(1, typeReadState.numShards());
26+
27+
HollowObjectTypeReadState typeReadStateSharded = populateTypeStateWith(5);
28+
assertDataUnchanged(typeReadStateSharded, 5);
29+
assertEquals(8, typeReadStateSharded.numShards());
30+
31+
HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(typeReadStateSharded.currentDataElements());
32+
HollowObjectTypeDataElements joinedDataElements = joiner.join();
33+
34+
typeReadState = new HollowObjectTypeReadState(typeReadState.getSchema(), joinedDataElements);
35+
assertDataUnchanged(typeReadState, 5);
36+
37+
try {
38+
joiner = new HollowObjectTypeDataElementsJoiner(mockObjectTypeState.currentDataElements());
39+
joiner.join();
40+
Assert.fail();
41+
} catch (IllegalStateException e) {
42+
// expected, numSplits should be a power of 2
43+
}
44+
}
45+
46+
@Test
47+
public void testJoinDifferentFieldWidths() throws IOException {
48+
HollowObjectTypeReadState typeReadStateSmall = populateTypeStateWith(new int[] {1});
49+
assertEquals(1, typeReadStateSmall.numShards());
50+
HollowObjectTypeDataElements dataElementsSmall = typeReadStateSmall.currentDataElements()[0];
51+
int intFieldPosSmall = dataElementsSmall.schema.getPosition("intField");
52+
int widthSmall = dataElementsSmall.bitsPerField[intFieldPosSmall];
53+
long valSmall = dataElementsSmall.fixedLengthData.getElementValue(dataElementsSmall.bitOffsetPerField[intFieldPosSmall], widthSmall);
54+
55+
HollowObjectTypeReadState typeReadStateBig = populateTypeStateWith(new int[] {2});
56+
assertEquals(1, typeReadStateBig.numShards());
57+
HollowObjectTypeDataElements dataElementsBig = typeReadStateBig.currentDataElements()[0];
58+
int intFieldPosBig = dataElementsBig.schema.getPosition("intField");
59+
int widthBig = dataElementsBig.bitsPerField[intFieldPosBig];
60+
long valBig = dataElementsBig.fixedLengthData.getElementValue(dataElementsBig.bitOffsetPerField[intFieldPosBig], widthBig);
61+
62+
assertTrue(widthBig > widthSmall);
63+
64+
HollowObjectTypeDataElementsJoiner joiner = new HollowObjectTypeDataElementsJoiner(new HollowObjectTypeDataElements[]
65+
{dataElementsSmall, dataElementsBig});
66+
HollowObjectTypeDataElements dataElementsJoined = joiner.join();
67+
int intFieldPosJoined = dataElementsJoined.schema.getPosition("intField");
68+
int widthJoined = dataElementsJoined.bitsPerField[intFieldPosJoined];
69+
70+
long val0 = dataElementsJoined.fixedLengthData.getElementValue(dataElementsJoined.bitOffsetPerField[intFieldPosJoined], widthJoined);
71+
long val1 = dataElementsJoined.fixedLengthData.getElementValue(dataElementsJoined.bitsPerRecord + dataElementsJoined.bitOffsetPerField[intFieldPosJoined], widthJoined);
72+
73+
assertEquals(widthBig, widthJoined);
74+
assertEquals(valSmall, val0);
75+
assertEquals(valBig, val1);
76+
}
77+
78+
@Test
79+
public void testLopsidedShards() {
80+
InMemoryBlobStore blobStore = new InMemoryBlobStore();
81+
HollowProducer p = HollowProducer.withPublisher(blobStore)
82+
.withBlobStager(new HollowInMemoryBlobStager())
83+
.withTypeResharding(true)
84+
.build();
85+
86+
p.initializeDataModel(schema);
87+
int targetSize = 64;
88+
p.getWriteEngine().setTargetMaxTypeShardSize(targetSize);
89+
long v1 = oneRunCycle(p, new int[] {0, 1, 2, 3, 4, 5, 6, 7});
90+
91+
HollowConsumer c = HollowConsumer
92+
.withBlobRetriever(blobStore)
93+
.withDoubleSnapshotConfig(new HollowConsumer.DoubleSnapshotConfig() {
94+
@Override
95+
public boolean allowDoubleSnapshot() {
96+
return false;
97+
}
98+
99+
@Override
100+
public int maxDeltasBeforeDoubleSnapshot() {
101+
return Integer.MAX_VALUE;
102+
}
103+
})
104+
.withSkipTypeShardUpdateWithNoAdditions()
105+
.build();
106+
c.triggerRefreshTo(v1);
107+
108+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards());
109+
assertEquals(true, c.getStateEngine().isSkipTypeShardUpdateWithNoAdditions());
110+
111+
long v2 = oneRunCycle(p, new int[] {0, 1, 2, 3, 5, 7});
112+
c.triggerRefreshTo(v2);
113+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards());
114+
115+
long v3 = oneRunCycle(p, new int[] { 0, 1, 3, 5}); // drop to 1 ordinal per shard, skipTypeShardWithNoAdds will make it so that maxOrdinal is adjusted
116+
c.triggerRefreshTo(v3);
117+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards());
118+
119+
long v4 = oneRunCycle(p, new int[] { 0, 1, 2, 3}); // now add another ordinal to one shard, maxOrdinals will be lopsided
120+
c.triggerRefreshTo(v4);
121+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards());
122+
123+
readStateEngine = c.getStateEngine();
124+
assertDataUnchanged(3);
125+
126+
long v5 = oneRunCycle(p, new int[] {0, 1});
127+
128+
// assert lopsided shards before join
129+
assertEquals(2, ((HollowObjectTypeReadState) c.getStateEngine().getTypeState("TestObject")).shardsVolatile.shards[0].dataElements.maxOrdinal);
130+
assertEquals(3, ((HollowObjectTypeReadState) c.getStateEngine().getTypeState("TestObject")).shardsVolatile.shards[1].dataElements.maxOrdinal);
131+
c.triggerRefreshTo(v5);
132+
assertEquals(1, c.getStateEngine().getTypeState("TestObject").numShards()); // joined to 1 shard
133+
readStateEngine = c.getStateEngine();
134+
assertDataUnchanged(2);
135+
136+
long v6 = oneRunCycle(p, new int[] {0, 1, 2, 3, 4, 5 });
137+
c.triggerRefreshTo(v6);
138+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards()); // split to 2 shards
139+
140+
long v7 = oneRunCycle(p, new int[] {8, 9});
141+
c.triggerRefreshTo(v7);
142+
assertEquals(4, c.getStateEngine().getTypeState("TestObject").numShards()); // still 2 shards
143+
144+
long v8 = oneRunCycle(p, new int[] {8});
145+
c.triggerRefreshTo(v8);
146+
assertEquals(2, c.getStateEngine().getTypeState("TestObject").numShards()); // down to 1 shard
147+
148+
c.triggerRefreshTo(v1);
149+
assertEquals(v1, c.getCurrentVersionId());
150+
151+
c.triggerRefreshTo(v8);
152+
assertEquals(v8, c.getCurrentVersionId());
153+
}
154+
155+
private long oneRunCycle(HollowProducer p, int recordIds[]) {
156+
return p.runCycle(state -> {
157+
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
158+
for(int recordId : recordIds) {
159+
rec.reset();
160+
rec.setLong("longField", recordId);
161+
rec.setString("stringField", "Value" + recordId);
162+
rec.setInt("intField", recordId);
163+
rec.setDouble("doubleField", recordId);
164+
165+
state.getStateEngine().add("TestObject", rec);
166+
}
167+
});
168+
}
169+
}

0 commit comments

Comments
 (0)