Skip to content

Commit 4709eb5

Browse files
SunjeetSunjeet
committed
Refactor tests, impl set split/join test
1 parent 3767beb commit 4709eb5

12 files changed

+175
-124
lines changed

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/set/HollowSetTypeReadState.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
*/
1717
package com.netflix.hollow.core.read.engine.set;
1818

19+
import static com.netflix.hollow.core.HollowConstants.ORDINAL_NONE;
20+
import static com.netflix.hollow.core.index.FieldPaths.FieldPathException.ErrorKind.NOT_BINDABLE;
21+
1922
import com.netflix.hollow.api.sampling.DisabledSamplingDirector;
2023
import com.netflix.hollow.api.sampling.HollowSampler;
2124
import com.netflix.hollow.api.sampling.HollowSamplingDirector;
@@ -46,9 +49,6 @@
4649
import java.util.logging.Level;
4750
import java.util.logging.Logger;
4851

49-
import static com.netflix.hollow.core.HollowConstants.ORDINAL_NONE;
50-
import static com.netflix.hollow.core.index.FieldPaths.FieldPathException.ErrorKind.NOT_BINDABLE;
51-
5252
/**
5353
* A {@link HollowTypeReadState} for OBJECT type records.
5454
*/
@@ -59,7 +59,7 @@ public class HollowSetTypeReadState extends HollowCollectionTypeReadState implem
5959

6060
private final int shardNumberMask;
6161
private final int shardOrdinalShift;
62-
private final HollowSetTypeReadStateShard shards[];
62+
final HollowSetTypeReadStateShard shards[]; // SNAP: TODO: elevated from private access for testing
6363

6464
private HollowPrimaryKeyValueDeriver keyDeriver;
6565

@@ -86,7 +86,20 @@ public HollowSetTypeReadState(HollowReadStateEngine stateEngine, MemoryMode memo
8686

8787
}
8888

89-
@Override
89+
// SNAP: TODO: for testing
90+
public HollowSetTypeReadState(HollowReadStateEngine stateEngine, MemoryMode memoryMode, HollowSetSchema schema, int numShards, HollowSetTypeReadStateShard[] shards) {
91+
super(stateEngine, memoryMode, schema);
92+
this.sampler = new HollowSetSampler(schema.getName(), DisabledSamplingDirector.INSTANCE);
93+
this.shardNumberMask = numShards - 1;
94+
this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(numShards);
95+
96+
if(numShards < 1 || 1 << shardOrdinalShift != numShards)
97+
throw new IllegalArgumentException("Number of shards must be a power of 2!");
98+
99+
this.shards = shards;
100+
}
101+
102+
@Override
90103
public void readSnapshot(HollowBlobInput in, ArraySegmentRecycler memoryRecycler, int numShards) throws IOException {
91104
throw new UnsupportedOperationException("This type does not yet support numShards specification when reading snapshot");
92105
}

‎hollow/src/test/java/com/netflix/hollow/core/read/HollowBlobOptionalPartTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.netflix.hollow.core.write.objectmapper.TypeA;
3535
import com.netflix.hollow.core.write.objectmapper.TypeB;
3636
import com.netflix.hollow.core.write.objectmapper.TypeC;
37+
import com.netflix.hollow.test.InMemoryBlobStore;
3738
import java.io.ByteArrayInputStream;
3839
import java.io.ByteArrayOutputStream;
3940
import java.io.File;
@@ -43,8 +44,6 @@
4344
import java.util.Arrays;
4445
import java.util.Collections;
4546
import java.util.HashSet;
46-
47-
import com.netflix.hollow.test.InMemoryBlobStore;
4847
import org.junit.Assert;
4948
import org.junit.Test;
5049

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitJoinTest.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package com.netflix.hollow.core.read.engine;
2+
3+
import com.netflix.hollow.core.AbstractStateEngineTest;
4+
import com.netflix.hollow.core.schema.HollowObjectSchema;
5+
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
6+
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
7+
import org.junit.Before;
8+
9+
public class AbstractHollowTypeDataElementsSplitJoinTest extends AbstractStateEngineTest {
10+
protected HollowObjectSchema schema;
11+
12+
@Before
13+
public void setUp() {
14+
schema = new HollowObjectSchema("TestObject", 4);
15+
schema.addField("longField", HollowObjectSchema.FieldType.LONG);
16+
schema.addField("stringField", HollowObjectSchema.FieldType.STRING);
17+
schema.addField("intField", HollowObjectSchema.FieldType.INT);
18+
schema.addField("doubleField", HollowObjectSchema.FieldType.DOUBLE);
19+
20+
super.setUp();
21+
}
22+
23+
@Override
24+
protected void initializeTypeStates() {
25+
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(schema));
26+
}
27+
28+
protected void populateWriteStateEngine(int numRecords) {
29+
initWriteStateEngine();
30+
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
31+
for(int i=0;i<numRecords;i++) {
32+
rec.reset();
33+
rec.setLong("longField", i);
34+
rec.setString("stringField", "Value" + i);
35+
rec.setInt("intField", i);
36+
rec.setDouble("doubleField", i);
37+
38+
writeStateEngine.add("TestObject", rec);
39+
}
40+
}
41+
42+
protected void populateWriteStateEngine(int[] recordIds) {
43+
initWriteStateEngine();
44+
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
45+
for(int recordId : recordIds) {
46+
rec.reset();
47+
rec.setLong("longField", recordId);
48+
rec.setString("stringField", "Value" + recordId);
49+
rec.setInt("intField", recordId);
50+
rec.setDouble("doubleField", recordId);
51+
52+
writeStateEngine.add("TestObject", rec);
53+
}
54+
}
55+
}

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java‎

Lines changed: 11 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,83 +2,43 @@
22

33
import static org.mockito.Mockito.when;
44

5-
import com.netflix.hollow.core.AbstractStateEngineTest;
6-
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
7-
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
5+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
86
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
97
import com.netflix.hollow.core.schema.HollowListSchema;
10-
import com.netflix.hollow.core.schema.HollowObjectSchema;
11-
import com.netflix.hollow.core.util.StateEngineRoundTripper;
128
import com.netflix.hollow.core.write.HollowListTypeWriteState;
139
import com.netflix.hollow.core.write.HollowListWriteRecord;
14-
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
15-
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
1610
import java.io.IOException;
1711
import java.util.Arrays;
1812
import org.junit.Assert;
1913
import org.junit.Before;
2014
import org.mockito.Mock;
2115
import org.mockito.MockitoAnnotations;
2216

23-
public class AbstractHollowListTypeDataElementsSplitJoinTest extends AbstractStateEngineTest {
24-
protected HollowObjectSchema objectSchema;
17+
public class AbstractHollowListTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
2518
protected HollowListSchema listSchema;
2619

2720
@Mock
2821
protected HollowListTypeReadState mockListTypeState;
2922

3023
@Before
3124
public void setUp() {
32-
this.objectSchema = new HollowObjectSchema("TestObject", 4);
33-
this.objectSchema.addField("longField", HollowObjectSchema.FieldType.LONG);
34-
this.objectSchema.addField("stringField", HollowObjectSchema.FieldType.STRING);
35-
this.objectSchema.addField("intField", HollowObjectSchema.FieldType.INT);
36-
this.objectSchema.addField("doubleField", HollowObjectSchema.FieldType.DOUBLE);
37-
3825
this.listSchema = new HollowListSchema("TestList", "TestObject");
3926

27+
super.setUp();
28+
4029
MockitoAnnotations.initMocks(this);
4130
HollowListTypeDataElements[] fakeDataElements = new HollowListTypeDataElements[5];
4231
when(mockListTypeState.currentDataElements()).thenReturn(fakeDataElements);
43-
super.setUp();
4432
}
4533

4634
@Override
4735
protected void initializeTypeStates() {
48-
writeStateEngine.setTargetMaxTypeShardSize(4096);
49-
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(objectSchema));
36+
super.initializeTypeStates();
5037
writeStateEngine.addTypeState(new HollowListTypeWriteState(listSchema));
38+
writeStateEngine.setTargetMaxTypeShardSize(4 * 100 * 1000 * 1024);
5139
}
5240

53-
private void populateWriteStateEngine(int numRecords, int[][] listContents) {
54-
initWriteStateEngine();
55-
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(objectSchema);
56-
for(int i=0;i<numRecords;i++) {
57-
rec.reset();
58-
rec.setLong("longField", i);
59-
rec.setString("stringField", "Value" + i);
60-
rec.setInt("intField", i);
61-
rec.setDouble("doubleField", i);
62-
63-
writeStateEngine.add("TestObject", rec);
64-
}
65-
for(int[] list : listContents) {
66-
addRecord(Arrays.stream(list).toArray());
67-
}
68-
}
69-
70-
private void populateWriteStateEngine(int[] recordIds, int[][] listContents) {
71-
initWriteStateEngine();
72-
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(objectSchema);
73-
for(int recordId : recordIds) {
74-
rec.reset();
75-
rec.setLong("longField", recordId);
76-
rec.setString("stringField", "Value" + recordId);
77-
rec.setInt("intField", recordId);
78-
rec.setDouble("doubleField", recordId);
79-
80-
writeStateEngine.add("TestObject", rec);
81-
}
41+
private void populateWriteStateEngine(int[][] listContents) {
8242
for(int[] list : listContents) {
8343
addRecord(Arrays.stream(list).toArray());
8444
}
@@ -96,13 +56,15 @@ private void addRecord(int... ordinals) {
9656

9757

9858
protected HollowListTypeReadState populateTypeStateWith(int numRecords, int[][] listContents) throws IOException {
99-
populateWriteStateEngine(numRecords, listContents);
59+
populateWriteStateEngine(numRecords);
60+
populateWriteStateEngine(listContents);
10061
roundTripSnapshot();
10162
return (HollowListTypeReadState) readStateEngine.getTypeState("TestList");
10263
}
10364

10465
protected HollowListTypeReadState populateTypeStateWith(int[] recordIds, int[][] listContents) throws IOException {
105-
populateWriteStateEngine(recordIds, listContents);
66+
populateWriteStateEngine(recordIds);
67+
populateWriteStateEngine(listContents);
10668
roundTripSnapshot();
10769
return (HollowListTypeReadState) readStateEngine.getTypeState("TestList");
10870
}

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitJoinTest.java‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@
66
import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever;
77
import com.netflix.hollow.core.memory.MemoryMode;
88
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
9-
import com.netflix.hollow.core.schema.HollowSchema;
10-
import com.netflix.hollow.core.write.HollowListTypeWriteState;
11-
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
129
import com.netflix.hollow.tools.checksum.HollowChecksum;
1310
import java.io.IOException;
1411
import java.nio.file.Paths;
@@ -17,13 +14,6 @@
1714

1815
public class HollowListTypeDataElementsSplitJoinTest extends AbstractHollowListTypeDataElementsSplitJoinTest {
1916

20-
@Override
21-
protected void initializeTypeStates() {
22-
writeStateEngine.setTargetMaxTypeShardSize(4 * 100 * 1000 * 1024);
23-
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(objectSchema));
24-
writeStateEngine.addTypeState(new HollowListTypeWriteState(listSchema));
25-
}
26-
2717
@Test
2818
public void testSplitThenJoin() throws IOException {
2919

@@ -80,7 +70,7 @@ private void assertChecksumUnchanged(HollowListTypeReadState newTypeState, Hollo
8070

8171
for(int i=0;i<origTypeState.numShards();i++) {
8272
origTypeState.shards[i].applyToChecksum(origCksum, populatedOrdinals, i, origTypeState.numShards());
83-
// SNAP: TODO: this will be shardsVolatile
73+
// SNAP: TODO: this will be shardsVolatile, and consider moving this method into base class
8474
}
8575

8676
for(int i=0;i<newTypeState.numShards();i++) {
@@ -131,7 +121,6 @@ public void testSplittingAndJoiningWithSnapshotBlob() throws Exception {
131121
throw new IllegalArgumentException("These arguments need to be specified");
132122
}
133123
HollowListTypeReadState typeState = (HollowListTypeReadState) readStateEngine.getTypeState(listTypeWithOneShard);
134-
HollowSchema origSchema = typeState.getSchema();
135124

136125
assertEquals(1, typeState.numShards());
137126

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/object/AbstractHollowObjectTypeDataElementsSplitJoinTest.java‎

Lines changed: 7 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,70 +4,33 @@
44
import static org.mockito.Mockito.when;
55

66
import com.netflix.hollow.api.objects.generic.GenericHollowObject;
7-
import com.netflix.hollow.core.AbstractStateEngineTest;
7+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitJoinTest;
88
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
99
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
10-
import com.netflix.hollow.core.schema.HollowObjectSchema;
1110
import com.netflix.hollow.core.util.StateEngineRoundTripper;
12-
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
13-
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
1411
import java.io.IOException;
1512
import org.junit.Before;
1613
import org.mockito.Mock;
1714
import org.mockito.MockitoAnnotations;
1815

19-
public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends AbstractStateEngineTest {
20-
protected HollowObjectSchema schema;
16+
public class AbstractHollowObjectTypeDataElementsSplitJoinTest extends AbstractHollowTypeDataElementsSplitJoinTest {
2117

2218
@Mock
2319
protected HollowObjectTypeReadState mockObjectTypeState;
2420

2521
@Before
2622
public void setUp() {
27-
schema = new HollowObjectSchema("TestObject", 4);
28-
schema.addField("longField", HollowObjectSchema.FieldType.LONG);
29-
schema.addField("stringField", HollowObjectSchema.FieldType.STRING);
30-
schema.addField("intField", HollowObjectSchema.FieldType.INT);
31-
schema.addField("doubleField", HollowObjectSchema.FieldType.DOUBLE);
23+
super.setUp();
3224

3325
MockitoAnnotations.initMocks(this);
3426
HollowObjectTypeDataElements[] fakeDataElements = new HollowObjectTypeDataElements[5];
3527
when(mockObjectTypeState.currentDataElements()).thenReturn(fakeDataElements);
36-
super.setUp();
3728
}
3829

3930
@Override
4031
protected void initializeTypeStates() {
41-
writeStateEngine.setTargetMaxTypeShardSize(4096);
42-
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(schema));
43-
}
44-
45-
private void populateWriteStateEngine(int numRecords) {
46-
initWriteStateEngine();
47-
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
48-
for(int i=0;i<numRecords;i++) {
49-
rec.reset();
50-
rec.setLong("longField", i);
51-
rec.setString("stringField", "Value" + i);
52-
rec.setInt("intField", i);
53-
rec.setDouble("doubleField", i);
54-
55-
writeStateEngine.add("TestObject", rec);
56-
}
57-
}
58-
59-
private void populateWriteStateEngine(int[] recordIds) {
60-
initWriteStateEngine();
61-
HollowObjectWriteRecord rec = new HollowObjectWriteRecord(schema);
62-
for(int recordId : recordIds) {
63-
rec.reset();
64-
rec.setLong("longField", recordId);
65-
rec.setString("stringField", "Value" + recordId);
66-
rec.setInt("intField", recordId);
67-
rec.setDouble("doubleField", recordId);
68-
69-
writeStateEngine.add("TestObject", rec);
70-
}
32+
super.initializeTypeStates();
33+
writeStateEngine.setTargetMaxTypeShardSize(4 * 1000 * 1024);
7134
}
7235

7336
protected HollowObjectTypeReadState populateTypeStateWith(int numRecords) throws IOException {
@@ -91,6 +54,8 @@ protected HollowObjectTypeReadState populateTypeStateWithFilter(int numRecords)
9154
return (HollowObjectTypeReadState) readStateEngine.getTypeState("TestObject");
9255
}
9356

57+
58+
9459
protected void assertDataUnchanged(int numRecords) {
9560
assertDataUnchanged((HollowObjectTypeReadState) readStateEngine.getTypeState("TestObject"), numRecords);
9661
}

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsJoinerTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import com.netflix.hollow.api.consumer.HollowConsumer;
77
import com.netflix.hollow.api.producer.HollowProducer;
88
import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
9-
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
109
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
1110
import com.netflix.hollow.test.InMemoryBlobStore;
1211
import java.io.IOException;
@@ -16,8 +15,8 @@
1615
public class HollowObjectTypeDataElementsJoinerTest extends AbstractHollowObjectTypeDataElementsSplitJoinTest {
1716
@Override
1817
protected void initializeTypeStates() {
18+
super.initializeTypeStates();
1919
writeStateEngine.setTargetMaxTypeShardSize(16);
20-
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(schema));
2120
}
2221

2322
@Test

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitJoinTest.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever;
77
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
88
import com.netflix.hollow.core.schema.HollowSchema;
9-
import com.netflix.hollow.core.write.HollowObjectTypeWriteState;
109
import com.netflix.hollow.tools.checksum.HollowChecksum;
1110
import java.io.IOException;
1211
import java.nio.file.Paths;
@@ -15,12 +14,6 @@
1514

1615
public class HollowObjectTypeDataElementsSplitJoinTest extends AbstractHollowObjectTypeDataElementsSplitJoinTest {
1716

18-
@Override
19-
protected void initializeTypeStates() {
20-
writeStateEngine.setTargetMaxTypeShardSize(4 * 1000 * 1024);
21-
writeStateEngine.addTypeState(new HollowObjectTypeWriteState(schema));
22-
}
23-
2417
@Test
2518
public void testSplitThenJoin() throws IOException {
2619
for (int numRecords=0;numRecords<1*1000;numRecords++) {

0 commit comments

Comments
 (0)