Skip to content

Commit a1d11de

Browse files
SunjeetSunjeet
committed
List split join data elements test
1 parent 37ccaf3 commit a1d11de

File tree

7 files changed

+266
-90
lines changed

7 files changed

+266
-90
lines changed

‎hollow/gradle.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.gradle.jvmargs=-Xmx30g

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsJoiner.java‎

Lines changed: 67 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,117 @@
1-
package com.netflix.hollow.core.read.engine.object;
2-
3-
import static com.netflix.hollow.core.read.engine.object.HollowObjectTypeDataElements.copyRecord;
4-
import static com.netflix.hollow.core.read.engine.object.HollowObjectTypeDataElements.varLengthSize;
5-
import static com.netflix.hollow.core.read.engine.object.HollowObjectTypeDataElements.writeNullField;
1+
package com.netflix.hollow.core.read.engine.list;
62

73
import com.netflix.hollow.core.memory.FixedLengthDataFactory;
8-
import com.netflix.hollow.core.memory.VariableLengthDataFactory;
94
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
105

116

127
/**
13-
* Join multiple {@code HollowObjectTypeDataElements}s into 1 {@code HollowObjectTypeDataElements}.
8+
* Join multiple {@code HollowListTypeDataElements}s into 1 {@code HollowListTypeDataElements}.
149
* Ordinals are remapped and corresponding data is copied over.
1510
* The original data elements are not destroyed.
1611
* The no. of passed data elements must be a power of 2.
1712
*/
18-
class HollowObjectTypeDataElementsJoiner {
13+
class HollowListTypeDataElementsJoiner {
1914

20-
HollowObjectTypeDataElements join(HollowObjectTypeDataElements[] from) {
15+
HollowListTypeDataElements join(HollowListTypeDataElements[] from) {
2116
final int fromMask = from.length - 1;
2217
final int fromOrdinalShift = 31 - Integer.numberOfLeadingZeros(from.length);
23-
long[] currentWriteVarLengthDataPointers;
2418

2519
if (from.length<=0 || !((from.length&(from.length-1))==0)) {
2620
throw new IllegalStateException("No. of DataElements to be joined must be a power of 2");
2721
}
2822

29-
HollowObjectTypeDataElements to = new HollowObjectTypeDataElements(from[0].schema, from[0].memoryMode, from[0].memoryRecycler);
30-
currentWriteVarLengthDataPointers = new long[from[0].schema.numFields()];
31-
32-
populateStats(to, from);
23+
HollowListTypeDataElements to = new HollowListTypeDataElements(from[0].memoryMode, from[0].memoryRecycler);
3324

3425
GapEncodedVariableLengthIntegerReader[] fromRemovals = new GapEncodedVariableLengthIntegerReader[from.length];
3526
for (int i=0;i<from.length;i++) {
3627
fromRemovals[i] = from[i].encodedRemovals;
3728
}
3829
to.encodedRemovals = GapEncodedVariableLengthIntegerReader.join(fromRemovals);
3930

40-
for (HollowObjectTypeDataElements elements : from) {
31+
for (HollowListTypeDataElements elements : from) {
4132
if (elements.encodedAdditions != null) {
4233
throw new IllegalStateException("Encountered encodedAdditions in data elements joiner- this is not expected " +
4334
"since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " +
4435
"delta data elements are never split/joined");
4536
}
4637
}
4738

48-
for(int ordinal=0;ordinal<=to.maxOrdinal;ordinal++) {
49-
int fromIndex = ordinal & fromMask;
50-
int fromOrdinal = ordinal >> fromOrdinalShift;
39+
populateStats(to, from);
5140

52-
if (fromOrdinal <= from[fromIndex].maxOrdinal) {
53-
copyRecord(to, ordinal, from[fromIndex], fromOrdinal, currentWriteVarLengthDataPointers);
54-
} else {
55-
// lopsided shards could result for consumers that skip type shards with no additions
56-
writeNullRecord(to, ordinal, currentWriteVarLengthDataPointers);
57-
}
58-
}
41+
copyRecords(to, from);
5942

6043
return to;
6144
}
6245

63-
private void writeNullRecord(HollowObjectTypeDataElements to, int toOrdinal, long[] currentWriteVarLengthDataPointers) {
64-
for(int fieldIndex=0;fieldIndex<to.schema.numFields();fieldIndex++) {
65-
long currentWriteFixedLengthStartBit = ((long)toOrdinal * to.bitsPerRecord) + to.bitOffsetPerField[fieldIndex];
66-
writeNullField(to, fieldIndex, currentWriteFixedLengthStartBit, currentWriteVarLengthDataPointers);
67-
}
68-
}
69-
70-
void populateStats(HollowObjectTypeDataElements to, HollowObjectTypeDataElements[] from) {
71-
long[] varLengthSizes = new long[to.schema.numFields()];
72-
46+
void populateStats(HollowListTypeDataElements to, HollowListTypeDataElements[] from) {
7347
to.maxOrdinal = -1;
7448
for(int fromIndex=0;fromIndex<from.length;fromIndex++) {
75-
for(int ordinal=0;ordinal<=from[fromIndex].maxOrdinal;ordinal++) {
76-
for(int fieldIdx=0;fieldIdx<to.schema.numFields();fieldIdx++) {
77-
if(from[fromIndex].varLengthData[fieldIdx] != null) {
78-
varLengthSizes[fieldIdx] += varLengthSize(from[fromIndex], ordinal, fieldIdx);
79-
}
80-
}
81-
}
8249
int mappedMaxOrdinal = from[fromIndex].maxOrdinal == -1 ? -1 : (from[fromIndex].maxOrdinal * from.length) + fromIndex;
8350
to.maxOrdinal = Math.max(to.maxOrdinal, mappedMaxOrdinal);
84-
}
85-
86-
for(int fieldIdx=0;fieldIdx<to.schema.numFields();fieldIdx++) {
87-
for(int i=0;i<from.length;i++) {
88-
if(from[i].varLengthData[fieldIdx] != null) { // if any of the join candidates have var len data set for this field
89-
to.varLengthData[fieldIdx] = VariableLengthDataFactory.get(to.memoryMode, to.memoryRecycler);
90-
break;
91-
}
51+
if (from[fromIndex].bitsPerElement > to.bitsPerElement) {
52+
// uneven bitsPerElement could be the case for consumers that skip type shards with no additions, so pick max across all shards
53+
to.bitsPerElement = from[fromIndex].bitsPerElement;
9254
}
9355
}
9456

95-
for(int fieldIdx=0;fieldIdx<to.schema.numFields();fieldIdx++) {
96-
if(to.varLengthData[fieldIdx] == null) {
97-
// do not assume bitsPerField will be uniform
98-
for(int fromIndex=0;fromIndex<from.length;fromIndex++) {
99-
to.bitsPerField[fieldIdx] = Math.max(to.bitsPerField[fieldIdx], from[fromIndex].bitsPerField[fieldIdx]);
100-
}
57+
final int fromMask = from.length - 1;
58+
final int fromOrdinalShift = 31 - Integer.numberOfLeadingZeros(from.length);
59+
long totalOfListSizes = 0;
60+
for(int ordinal=0;ordinal<=to.maxOrdinal;ordinal++) {
61+
int fromIndex = ordinal & fromMask;
62+
int fromOrdinal = ordinal >> fromOrdinalShift;
63+
64+
long startElement;
65+
long endElement;
66+
if (fromOrdinal == 0) {
67+
startElement = 0;
68+
endElement = from[fromIndex].listPointerData.getElementValue(0, from[fromIndex].bitsPerListPointer);
10169
} else {
102-
to.bitsPerField[fieldIdx] = (64 - Long.numberOfLeadingZeros(varLengthSizes[fieldIdx] + 1)) + 1;
70+
long endFixedLengthOffset = (long)fromOrdinal * from[fromIndex].bitsPerListPointer;
71+
long startFixedLengthOffset = endFixedLengthOffset - from[fromIndex].bitsPerListPointer;
72+
startElement = from[fromIndex].listPointerData.getElementValue(startFixedLengthOffset, from[fromIndex].bitsPerListPointer);
73+
endElement = from[fromIndex].listPointerData.getElementValue(endFixedLengthOffset, from[fromIndex].bitsPerListPointer);
10374
}
104-
to.nullValueForField[fieldIdx] = to.bitsPerField[fieldIdx] == 64 ? -1L : (1L << to.bitsPerField[fieldIdx]) - 1;
105-
to.bitOffsetPerField[fieldIdx] = to.bitsPerRecord;
106-
to.bitsPerRecord += to.bitsPerField[fieldIdx];
75+
long numElements = endElement - startElement;
76+
totalOfListSizes += numElements;
77+
10778
}
108-
to.fixedLengthData = FixedLengthDataFactory.get((long)to.bitsPerRecord * (to.maxOrdinal + 1), to.memoryMode, to.memoryRecycler);
79+
to.bitsPerListPointer = totalOfListSizes == 0 ? 1 : 64 - Long.numberOfLeadingZeros(totalOfListSizes);
80+
to.listPointerData = FixedLengthDataFactory.get((long)to.bitsPerListPointer * (to.maxOrdinal + 1), to.memoryMode, to.memoryRecycler);
81+
to.elementData = FixedLengthDataFactory.get((long)to.bitsPerElement * totalOfListSizes, to.memoryMode, to.memoryRecycler);
82+
to.totalNumberOfElements = totalOfListSizes;
83+
}
84+
85+
private void copyRecords(HollowListTypeDataElements to, HollowListTypeDataElements[] from) {
86+
final int fromMask = from.length - 1;
87+
final int fromOrdinalShift = 31 - Integer.numberOfLeadingZeros(from.length);
88+
long elementCounter = 0;
89+
for(int ordinal=0;ordinal<=to.maxOrdinal;ordinal++) {
90+
int fromIndex = ordinal & fromMask;
91+
int fromOrdinal = ordinal >> fromOrdinalShift;
10992

110-
// unused
111-
// to.bitsPerUnfilteredField
112-
// to.unfilteredFieldIsIncluded
93+
long startElement;
94+
long endElement;
95+
if (fromOrdinal <= from[fromIndex].maxOrdinal) {
96+
if (fromOrdinal == 0) {
97+
startElement = 0;
98+
endElement = from[fromIndex].listPointerData.getElementValue(0, from[fromIndex].bitsPerListPointer);
99+
} else {
100+
long endFixedLengthOffset = (long)fromOrdinal * from[fromIndex].bitsPerListPointer;
101+
long startFixedLengthOffset = endFixedLengthOffset - from[fromIndex].bitsPerListPointer;
102+
startElement = from[fromIndex].listPointerData.getElementValue(startFixedLengthOffset, from[fromIndex].bitsPerListPointer);
103+
endElement = from[fromIndex].listPointerData.getElementValue(endFixedLengthOffset, from[fromIndex].bitsPerListPointer);
104+
}
105+
for (long element=startElement;element<endElement;element++) {
106+
int elementOrdinal = (int)from[fromIndex].elementData.getElementValue(element * from[fromIndex].bitsPerElement, from[fromIndex].bitsPerElement);
107+
to.elementData.setElementValue(elementCounter * to.bitsPerElement, to.bitsPerElement, elementOrdinal);
108+
elementCounter++;
109+
}
110+
} // else: lopsided shards could result for consumers that skip type shards with no additions, that gets handled
111+
// by not writing anything to elementData, and writing the cached value of elementCounter to listPointerData
112+
// SNAP: TODO: write a test for lopsided list shards
113+
114+
to.listPointerData.setElementValue(to.bitsPerListPointer * ordinal, to.bitsPerListPointer, elementCounter);
115+
}
113116
}
114117
}

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ HollowListTypeDataElements[] split(HollowListTypeDataElements from, int numSplit
3939

4040
populateStats(to, from, toMask, toOrdinalShift);
4141

42-
copyRecords(to, from, toMask);
42+
copyRecords(to, from, toMask, toOrdinalShift);
4343

4444
return to;
4545
}
@@ -82,19 +82,19 @@ private void populateStats(HollowListTypeDataElements[] to, HollowListTypeDataEl
8282

8383
to[toIndex].listPointerData = FixedLengthDataFactory.get((long)to[toIndex].bitsPerListPointer * (to[toIndex].maxOrdinal + 1), to[toIndex].memoryMode, to[toIndex].memoryRecycler);
8484
to[toIndex].elementData = FixedLengthDataFactory.get((long)to[toIndex].bitsPerElement * totalOfListSizes[toIndex], to[toIndex].memoryMode, to[toIndex].memoryRecycler);
85-
// SNAP: TODO: who does clean up?
8685

8786
to[toIndex].totalNumberOfElements = totalOfListSizes[toIndex]; // useful for heap usage stats
8887
}
8988
}
9089

91-
private void copyRecords(HollowListTypeDataElements[] to, HollowListTypeDataElements from, int toMask) {
90+
private void copyRecords(HollowListTypeDataElements[] to, HollowListTypeDataElements from, int toMask, int toOrdinalShift) {
9291
int numSplits = to.length;
9392
long elementCounter[] = new long[numSplits];
9493

9594
// count elements per split
9695
for(int ordinal=0;ordinal<=from.maxOrdinal;ordinal++) {
9796
int toIndex = ordinal & toMask;
97+
int toOrdinal = ordinal >> toOrdinalShift;
9898

9999
long startElement;
100100
long endElement;
@@ -113,7 +113,7 @@ private void copyRecords(HollowListTypeDataElements[] to, HollowListTypeDataElem
113113
to[toIndex].elementData.setElementValue(elementCounter[toIndex] * to[toIndex].bitsPerElement, to[toIndex].bitsPerElement, elementOrdinal);
114114
elementCounter[toIndex]++;
115115
}
116-
to[toIndex].listPointerData.setElementValue(to[toIndex].bitsPerListPointer * toIndex, to[toIndex].bitsPerListPointer, elementCounter[toIndex]);
116+
to[toIndex].listPointerData.setElementValue(to[toIndex].bitsPerListPointer * toOrdinal, to[toIndex].bitsPerListPointer, elementCounter[toIndex]);
117117
}
118118
}
119119
}

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeReadState.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ public class HollowListTypeReadState extends HollowCollectionTypeReadState imple
4747

4848
private final HollowListSampler sampler;
4949

50-
private final int shardNumberMask;
50+
final int shardNumberMask; // SNAP: TODO: elevated from private access for testing
5151
private final int shardOrdinalShift;
52-
private final HollowListTypeReadStateShard shards[];
52+
final HollowListTypeReadStateShard shards[]; // SNAP: TODO: elevated from private access for testing
5353

5454
private int maxOrdinal;
5555

@@ -73,6 +73,19 @@ public HollowListTypeReadState(HollowReadStateEngine stateEngine, MemoryMode mem
7373
this.shards = shards;
7474
}
7575

76+
// SNAP: TODO: for testing
77+
public HollowListTypeReadState(HollowReadStateEngine stateEngine, MemoryMode memoryMode, HollowListSchema schema, int numShards, HollowListTypeReadStateShard[] shards) {
78+
super(stateEngine, memoryMode, schema);
79+
this.sampler = new HollowListSampler(schema.getName(), DisabledSamplingDirector.INSTANCE);
80+
this.shardNumberMask = numShards - 1;
81+
this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(numShards);
82+
83+
if(numShards < 1 || 1 << shardOrdinalShift != numShards)
84+
throw new IllegalArgumentException("Number of shards must be a power of 2!");
85+
86+
this.shards = shards;
87+
}
88+
7689
@Override
7790
public void readSnapshot(HollowBlobInput in, ArraySegmentRecycler memoryRecycler, int numShards) throws IOException {
7891
throw new UnsupportedOperationException("This type does not yet support numShards specification when reading snapshot");

‎hollow/src/test/java/com/netflix/hollow/core/read/engine/list/AbstractHollowListTypeDataElementsSplitJoinTest.java‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
package com.netflix.hollow.core.read.engine.list;
22

3-
import static org.junit.Assert.assertEquals;
43
import static org.mockito.Mockito.when;
54

6-
import com.netflix.hollow.api.objects.generic.GenericHollowList;
7-
import com.netflix.hollow.api.objects.generic.GenericHollowObject;
85
import com.netflix.hollow.core.AbstractStateEngineTest;
96
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
107
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
8+
import com.netflix.hollow.core.read.iterator.HollowOrdinalIterator;
119
import com.netflix.hollow.core.schema.HollowListSchema;
1210
import com.netflix.hollow.core.schema.HollowObjectSchema;
1311
import com.netflix.hollow.core.util.StateEngineRoundTripper;
@@ -17,6 +15,7 @@
1715
import com.netflix.hollow.core.write.HollowObjectWriteRecord;
1816
import java.io.IOException;
1917
import java.util.Arrays;
18+
import org.junit.Assert;
2019
import org.junit.Before;
2120
import org.mockito.Mock;
2221
import org.mockito.MockitoAnnotations;
@@ -108,23 +107,19 @@ protected HollowListTypeReadState populateTypeStateWith(int[] recordIds, int[][]
108107
return (HollowListTypeReadState) readStateEngine.getTypeState("TestList");
109108
}
110109

111-
protected HollowListTypeReadState populateTypeStateWithFilter(int numRecords, int[][] listContents) throws IOException {
112-
populateWriteStateEngine(numRecords, listContents);
113-
readStateEngine = new HollowReadStateEngine();
114-
HollowFilterConfig readFilter = new HollowFilterConfig(true);
115-
readFilter.addField("TestObject", "intField");
116-
StateEngineRoundTripper.roundTripSnapshot(writeStateEngine, readStateEngine, readFilter);
117-
return (HollowListTypeReadState) readStateEngine.getTypeState("TestList");
110+
protected void assertDataUnchanged(int[][] listContents) {
111+
assertDataUnchanged((HollowListTypeReadState) readStateEngine.getTypeState("TestList"), listContents);
118112
}
119113

120-
protected void assertDataUnchanged(int numRecords) {
121-
assertDataUnchanged((HollowListTypeReadState) readStateEngine.getTypeState("TestObject"), numRecords);
122-
}
123-
124-
protected void assertDataUnchanged(HollowListTypeReadState typeState, int numRecords) {
125-
for(int i=0;i<numRecords;i++) {
126-
GenericHollowList obj = new GenericHollowList(typeState, i);
127-
System.out.println(obj.toString());
114+
protected void assertDataUnchanged(HollowListTypeReadState typeState, int[][] listContents) {
115+
int numListRecords = listContents.length;
116+
for(int i=0;i<numListRecords;i++) {
117+
HollowOrdinalIterator iter = typeState.ordinalIterator(i);
118+
for(int j=0;j<listContents[i].length;j++) {
119+
Assert.assertEquals(listContents[i][j], iter.next());
120+
}
121+
Assert.assertEquals(HollowOrdinalIterator.NO_MORE_ORDINALS, iter.next());
122+
// System.out.println(obj.toString());
128123
// assertEquals(i, obj.get("longField"));
129124
// assertEquals("Value"+i, obj.getString("stringField"));
130125
// assertEquals((double)i, obj.getDouble("doubleField"), 0);

0 commit comments

Comments
 (0)