Skip to content

Commit 70f3614

Browse files
SunjeetSunjeet
committed
Refactor data elements and splitter for polymorphism
1 parent a1d11de commit 70f3614

14 files changed

+197
-150
lines changed

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElements.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.netflix.hollow.core.read.engine;
2+
3+
import com.netflix.hollow.core.memory.MemoryMode;
4+
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
5+
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
6+
7+
public abstract class AbstractHollowTypeDataElements {
8+
9+
public int maxOrdinal;
10+
public GapEncodedVariableLengthIntegerReader encodedAdditions;
11+
public GapEncodedVariableLengthIntegerReader encodedRemovals;
12+
public final ArraySegmentRecycler memoryRecycler;
13+
public final MemoryMode memoryMode;
14+
15+
public AbstractHollowTypeDataElements(MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler) {
16+
this.memoryMode = memoryMode;
17+
this.memoryRecycler = memoryRecycler;
18+
}
19+
}

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/AbstractHollowTypeDataElementsSplitter.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.netflix.hollow.core.read.engine;
2+
3+
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
4+
5+
public abstract class AbstractHollowTypeDataElementsSplitter {
6+
public final int numSplits;
7+
public final int toMask;
8+
public final int toOrdinalShift;
9+
public final AbstractHollowTypeDataElements from;
10+
11+
public AbstractHollowTypeDataElements[] to;
12+
13+
public AbstractHollowTypeDataElementsSplitter(AbstractHollowTypeDataElements from, int numSplits) {
14+
this.from = from;
15+
this.numSplits = numSplits;
16+
this.toMask = numSplits - 1;
17+
this.toOrdinalShift = 31 - Integer.numberOfLeadingZeros(numSplits);
18+
19+
if (numSplits<=0 || !((numSplits&(numSplits-1))==0)) {
20+
throw new IllegalStateException("Must split by power of 2");
21+
}
22+
23+
if (from.encodedAdditions != null) {
24+
throw new IllegalStateException("Encountered encodedAdditions in data elements splitter- this is not expected " +
25+
"since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " +
26+
"delta data elements are never split/joined");
27+
}
28+
}
29+
30+
public AbstractHollowTypeDataElements[] split() {
31+
for(int i=0;i<to.length;i++) {
32+
to[i].maxOrdinal = -1;
33+
}
34+
populateStats();
35+
36+
copyRecords();
37+
38+
if (from.encodedRemovals != null) {
39+
GapEncodedVariableLengthIntegerReader[] splitRemovals = from.encodedRemovals.split(numSplits);
40+
for(int i=0;i<to.length;i++) {
41+
to[i].encodedRemovals = splitRemovals[i];
42+
}
43+
}
44+
45+
return to;
46+
}
47+
48+
public abstract void populateStats();
49+
50+
public abstract void copyRecords();
51+
52+
53+
}

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElements.java

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.netflix.hollow.core.memory.encoding.VarInt;
2424
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
2525
import com.netflix.hollow.core.read.HollowBlobInput;
26+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElements;
2627
import java.io.IOException;
2728

2829
/**
@@ -31,30 +32,21 @@
3132
* During a delta, the HollowListTypeReadState will create a new HollowListTypeDataElements and atomically swap
3233
* with the existing one to make sure a consistent view of the data is always available.
3334
*/
34-
public class HollowListTypeDataElements {
35-
36-
int maxOrdinal;
35+
public class HollowListTypeDataElements extends AbstractHollowTypeDataElements {
3736

3837
FixedLengthData listPointerData;
3938
FixedLengthData elementData;
4039

41-
GapEncodedVariableLengthIntegerReader encodedAdditions;
42-
GapEncodedVariableLengthIntegerReader encodedRemovals;
43-
4440
int bitsPerListPointer;
4541
int bitsPerElement;
4642
long totalNumberOfElements = 0;
4743

48-
final ArraySegmentRecycler memoryRecycler;
49-
final MemoryMode memoryMode;
50-
5144
public HollowListTypeDataElements(ArraySegmentRecycler memoryRecycler) {
5245
this(MemoryMode.ON_HEAP, memoryRecycler);
5346
}
5447

5548
public HollowListTypeDataElements(MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler) {
56-
this.memoryMode = memoryMode;
57-
this.memoryRecycler = memoryRecycler;
49+
super(memoryMode, memoryRecycler);
5850
}
5951

6052
void readSnapshot(HollowBlobInput in) throws IOException {

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/list/HollowListTypeDataElementsSplitter.java

Lines changed: 15 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,29 @@
11
package com.netflix.hollow.core.read.engine.list;
22

33
import com.netflix.hollow.core.memory.FixedLengthDataFactory;
4-
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
4+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitter;
55

66
/**
77
* Split a {@code HollowListTypeDataElements} into multiple {@code HollowListTypeDataElements}s.
88
* Ordinals are remapped and corresponding data is copied over.
99
* The original data elements are not destroyed.
1010
* {@code numSplits} must be a power of 2.
1111
*/
12-
public class HollowListTypeDataElementsSplitter {
12+
public class HollowListTypeDataElementsSplitter extends AbstractHollowTypeDataElementsSplitter {
1313

14-
HollowListTypeDataElements[] split(HollowListTypeDataElements from, int numSplits) {
15-
final int toMask = numSplits - 1;
16-
final int toOrdinalShift = 31 - Integer.numberOfLeadingZeros(numSplits);
17-
18-
if (numSplits<=0 || !((numSplits&(numSplits-1))==0)) {
19-
throw new IllegalStateException("Must split by power of 2");
20-
}
21-
22-
HollowListTypeDataElements[] to = new HollowListTypeDataElements[numSplits];
14+
public HollowListTypeDataElementsSplitter(HollowListTypeDataElements from, int numSplits) {
15+
super(from, numSplits);
16+
this.to = new HollowListTypeDataElements[numSplits];
2317
for(int i=0;i<to.length;i++) {
2418
to[i] = new HollowListTypeDataElements(from.memoryMode, from.memoryRecycler);
25-
to[i].maxOrdinal = -1;
26-
}
27-
28-
if (from.encodedRemovals != null) {
29-
GapEncodedVariableLengthIntegerReader[] splitRemovals = from.encodedRemovals.split(numSplits);
30-
for(int i=0;i<to.length;i++) {
31-
to[i].encodedRemovals = splitRemovals[i];
32-
}
3319
}
34-
if (from.encodedAdditions != null) {
35-
throw new IllegalStateException("Encountered encodedAdditions in data elements splitter- this is not expected " +
36-
"since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " +
37-
"delta data elements are never split/joined");
38-
}
39-
40-
populateStats(to, from, toMask, toOrdinalShift);
41-
42-
copyRecords(to, from, toMask, toOrdinalShift);
43-
44-
return to;
4520
}
4621

47-
private void populateStats(HollowListTypeDataElements[] to, HollowListTypeDataElements from, int toMask, int toOrdinalShift) {
22+
@Override
23+
public void populateStats() {
24+
HollowListTypeDataElements[] to = (HollowListTypeDataElements[]) this.to;
25+
HollowListTypeDataElements from = (HollowListTypeDataElements) this.from;
26+
4827
int numSplits = to.length;
4928
long[] totalOfListSizes = new long[numSplits];
5029

@@ -87,7 +66,11 @@ private void populateStats(HollowListTypeDataElements[] to, HollowListTypeDataEl
8766
}
8867
}
8968

90-
private void copyRecords(HollowListTypeDataElements[] to, HollowListTypeDataElements from, int toMask, int toOrdinalShift) {
69+
@Override
70+
public void copyRecords() {
71+
HollowListTypeDataElements[] to = (HollowListTypeDataElements[]) this.to;
72+
HollowListTypeDataElements from = (HollowListTypeDataElements) this.from;
73+
9174
int numSplits = to.length;
9275
long elementCounter[] = new long[numSplits];
9376

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/map/HollowMapTypeDataElements.java

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.netflix.hollow.core.memory.encoding.VarInt;
2424
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
2525
import com.netflix.hollow.core.read.HollowBlobInput;
26+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElements;
2627
import java.io.IOException;
2728

2829
/**
@@ -31,16 +32,11 @@
3132
* During a delta, the HollowMapTypeReadState will create a new HollowMapTypeDataElements and atomically swap
3233
* with the existing one to make sure a consistent view of the data is always available.
3334
*/
34-
public class HollowMapTypeDataElements {
35-
36-
int maxOrdinal;
35+
public class HollowMapTypeDataElements extends AbstractHollowTypeDataElements {
3736

3837
FixedLengthData mapPointerAndSizeData;
3938
FixedLengthData entryData;
4039

41-
GapEncodedVariableLengthIntegerReader encodedRemovals;
42-
GapEncodedVariableLengthIntegerReader encodedAdditions;
43-
4440
int bitsPerMapPointer;
4541
int bitsPerMapSizeValue;
4642
int bitsPerFixedLengthMapPortion;
@@ -50,16 +46,12 @@ public class HollowMapTypeDataElements {
5046
int emptyBucketKeyValue;
5147
long totalNumberOfBuckets;
5248

53-
final ArraySegmentRecycler memoryRecycler;
54-
final MemoryMode memoryMode;
55-
5649
public HollowMapTypeDataElements(ArraySegmentRecycler memoryRecycler) {
5750
this(MemoryMode.ON_HEAP, memoryRecycler);
5851
}
5952

6053
public HollowMapTypeDataElements(MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler) {
61-
this.memoryMode = memoryMode;
62-
this.memoryRecycler = memoryRecycler;
54+
super(memoryMode, memoryRecycler);
6355
}
6456

6557
void readSnapshot(HollowBlobInput in) throws IOException {

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElements.java

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.netflix.hollow.core.memory.encoding.VarInt;
2727
import com.netflix.hollow.core.memory.pool.ArraySegmentRecycler;
2828
import com.netflix.hollow.core.read.HollowBlobInput;
29+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElements;
2930
import com.netflix.hollow.core.schema.HollowObjectSchema;
3031
import java.io.IOException;
3132

@@ -35,18 +36,13 @@
3536
* During a delta, the HollowObjectTypeReadState will create a new HollowObjectTypeDataElements and atomically swap
3637
* with the existing one to make sure a consistent view of the data is always available.
3738
*/
38-
public class HollowObjectTypeDataElements {
39+
public class HollowObjectTypeDataElements extends AbstractHollowTypeDataElements {
3940

40-
final HollowObjectSchema schema;
41-
42-
int maxOrdinal;
41+
public final HollowObjectSchema schema;
4342

4443
FixedLengthData fixedLengthData;
4544
final VariableLengthData varLengthData[];
4645

47-
GapEncodedVariableLengthIntegerReader encodedAdditions;
48-
GapEncodedVariableLengthIntegerReader encodedRemovals;
49-
5046
final int bitsPerField[];
5147
final int bitOffsetPerField[];
5248
final long nullValueForField[];
@@ -55,21 +51,17 @@ public class HollowObjectTypeDataElements {
5551
private int bitsPerUnfilteredField[];
5652
private boolean unfilteredFieldIsIncluded[];
5753

58-
final ArraySegmentRecycler memoryRecycler;
59-
final MemoryMode memoryMode;
60-
6154
public HollowObjectTypeDataElements(HollowObjectSchema schema, ArraySegmentRecycler memoryRecycler) {
6255
this(schema, MemoryMode.ON_HEAP, memoryRecycler);
6356
}
6457

6558
public HollowObjectTypeDataElements(HollowObjectSchema schema, MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler) {
59+
super(memoryMode, memoryRecycler);
6660
varLengthData = new VariableLengthData[schema.numFields()];
6761
bitsPerField = new int[schema.numFields()];
6862
bitOffsetPerField = new int[schema.numFields()];
6963
nullValueForField = new long[schema.numFields()];
7064
this.schema = schema;
71-
this.memoryMode = memoryMode;
72-
this.memoryRecycler = memoryRecycler;
7365
}
7466

7567
void readSnapshot(HollowBlobInput in, HollowObjectSchema unfilteredSchema) throws IOException {

‎hollow/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeDataElementsSplitter.java

Lines changed: 35 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,64 +5,31 @@
55

66
import com.netflix.hollow.core.memory.FixedLengthDataFactory;
77
import com.netflix.hollow.core.memory.VariableLengthDataFactory;
8-
import com.netflix.hollow.core.memory.encoding.GapEncodedVariableLengthIntegerReader;
8+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElements;
9+
import com.netflix.hollow.core.read.engine.AbstractHollowTypeDataElementsSplitter;
910

1011
/**
1112
* Split a {@code HollowObjectTypeDataElements} into multiple {@code HollowObjectTypeDataElements}s.
1213
* Ordinals are remapped and corresponding data is copied over.
1314
* The original data elements are not destroyed.
15+
* // SNAP: TODO: single-shot, split() not expected to be called repeatedly
1416
* {@code numSplits} must be a power of 2.
1517
*/
16-
public class HollowObjectTypeDataElementsSplitter {
18+
public class HollowObjectTypeDataElementsSplitter extends AbstractHollowTypeDataElementsSplitter {
1719

18-
HollowObjectTypeDataElements[] split(HollowObjectTypeDataElements from, int numSplits) {
19-
final int toMask = numSplits - 1;
20-
final int toOrdinalShift = 31 - Integer.numberOfLeadingZeros(numSplits);
21-
final long[][] currentWriteVarLengthDataPointers;
22-
23-
if (numSplits<=0 || !((numSplits&(numSplits-1))==0)) {
24-
throw new IllegalStateException("Must split by power of 2");
25-
}
26-
27-
HollowObjectTypeDataElements[] to = new HollowObjectTypeDataElements[numSplits];
20+
HollowObjectTypeDataElementsSplitter(HollowObjectTypeDataElements from, int numSplits) {
21+
super(from, numSplits);
22+
this.to = new HollowObjectTypeDataElements[numSplits];
2823
for(int i=0;i<to.length;i++) {
2924
to[i] = new HollowObjectTypeDataElements(from.schema, from.memoryMode, from.memoryRecycler);
30-
to[i].maxOrdinal = -1;
31-
}
32-
currentWriteVarLengthDataPointers = new long[numSplits][from.schema.numFields()];
33-
34-
populateStats(to, from, toMask, toOrdinalShift);
35-
36-
if (from.encodedRemovals != null) {
37-
GapEncodedVariableLengthIntegerReader[] splitRemovals = from.encodedRemovals.split(numSplits);
38-
for(int i=0;i<to.length;i++) {
39-
to[i].encodedRemovals = splitRemovals[i];
40-
}
41-
}
42-
if (from.encodedAdditions != null) {
43-
throw new IllegalStateException("Encountered encodedAdditions in data elements splitter- this is not expected " +
44-
"since encodedAdditions only exist on delta data elements and they dont carry over to target data elements, " +
45-
"delta data elements are never split/joined");
46-
}
47-
48-
for(int i=0;i<to.length;i++) {
49-
to[i].fixedLengthData = FixedLengthDataFactory.get((long)to[i].bitsPerRecord * (to[i].maxOrdinal + 1), to[i].memoryMode, to[i].memoryRecycler);
50-
for(int fieldIdx=0;fieldIdx<from.schema.numFields();fieldIdx++) {
51-
if(from.varLengthData[fieldIdx] != null) {
52-
to[i].varLengthData[fieldIdx] = VariableLengthDataFactory.get(from.memoryMode, from.memoryRecycler);
53-
}
54-
}
55-
}
56-
57-
for(int i=0;i<=from.maxOrdinal;i++) {
58-
int toIndex = i & toMask;
59-
int toOrdinal = i >> toOrdinalShift;
60-
copyRecord(to[toIndex], toOrdinal, from, i, currentWriteVarLengthDataPointers[toIndex]);
6125
}
62-
return to;
6326
}
6427

65-
private void populateStats(HollowObjectTypeDataElements[] to, HollowObjectTypeDataElements from, int toMask, int toOrdinalShift) {
28+
@Override
29+
public void populateStats() {
30+
HollowObjectTypeDataElements[] to = (HollowObjectTypeDataElements[])this.to;
31+
HollowObjectTypeDataElements from = (HollowObjectTypeDataElements) this.from;
32+
6633
long[][] varLengthSizes = new long[to.length][from.schema.numFields()];
6734

6835
for(int ordinal=0;ordinal<=from.maxOrdinal;ordinal++) {
@@ -93,4 +60,27 @@ private void populateStats(HollowObjectTypeDataElements[] to, HollowObjectTypeDa
9360
}
9461
}
9562
}
63+
64+
@Override
65+
public void copyRecords() {
66+
HollowObjectTypeDataElements[] to = (HollowObjectTypeDataElements[])this.to;
67+
HollowObjectTypeDataElements from = (HollowObjectTypeDataElements) this.from;
68+
69+
final long[][] currentWriteVarLengthDataPointers = new long[to.length][from.schema.numFields()];
70+
71+
for(int i=0;i<to.length;i++) {
72+
to[i].fixedLengthData = FixedLengthDataFactory.get((long)to[i].bitsPerRecord * (to[i].maxOrdinal + 1), to[i].memoryMode, to[i].memoryRecycler);
73+
for(int fieldIdx=0;fieldIdx<from.schema.numFields();fieldIdx++) {
74+
if(from.varLengthData[fieldIdx] != null) {
75+
to[i].varLengthData[fieldIdx] = VariableLengthDataFactory.get(from.memoryMode, from.memoryRecycler);
76+
}
77+
}
78+
}
79+
80+
for(int i=0;i<=from.maxOrdinal;i++) {
81+
int toIndex = i & toMask;
82+
int toOrdinal = i >> toOrdinalShift;
83+
copyRecord(to[toIndex], toOrdinal, from, i, currentWriteVarLengthDataPointers[toIndex]);
84+
}
85+
}
9686
}

0 commit comments

Comments
 (0)