Skip to content

Commit 1fd1a84

Browse files
SunjeetSunjeet
authored
Producer validator for min record count (#737)
* Producer validator for min record count * Add a mention of the min record count validator to user docs
1 parent 325c60c commit 1fd1a84

File tree

4 files changed

+177
-15
lines changed

4 files changed

+177
-15
lines changed

‎docs/validation.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ unexpectedly decreases or increases.
109109
3. An object modification validator, `ObjectModificationValidator`.
110110
This validator can be configured to compare the state of objects (with the same primary key)
111111
that have been modified (not added or removed).
112+
4. A minimum record count validator, `MinimumRecordCountValidator`.
113+
This validator can be configured to guard against large deletions, by setting a threshold for
114+
allowable minimum number of records in a data type.
112115

113116

114117
## Using the generated object API

‎hollow/src/main/java/com/netflix/hollow/api/producer/validation/MinimumRecordCountValidator.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package com.netflix.hollow.api.producer.validation;
2+
3+
import com.netflix.hollow.api.producer.HollowProducer;
4+
import com.netflix.hollow.core.read.engine.HollowTypeReadState;
5+
import java.util.function.Supplier;
6+
7+
public class MinimumRecordCountValidator implements ValidatorListener {
8+
9+
private static final String INVALID_THRESHOLD =
10+
"Minimum record count validation for type %s has failed due to invalid threshold: %s";
11+
private static final String FAILED_MIN_RECORD_COUNT_VALIDATION =
12+
"Minimum record count validation for type %s has failed since the record count %s is less than the "
13+
+ "configured threshold of %s";
14+
15+
private static final String DATA_TYPE_NAME = "Typename";
16+
private static final String ALLOWABLE_MIN_RECORD_COUNT_NAME = "AllowableMinRecordCount";
17+
private static final String RECORD_COUNT_NAME = "RecordCount";
18+
19+
private final String typeName;
20+
private final Supplier<Integer> minRecordCountSupplier;
21+
22+
public MinimumRecordCountValidator(String typeName, Supplier<Integer> minRecordCountSupplier) {
23+
this.typeName = typeName;
24+
this.minRecordCountSupplier = minRecordCountSupplier;
25+
}
26+
27+
@Override
28+
public String getName() {
29+
return MinimumRecordCountValidator.class.getName() + "_" + typeName;
30+
}
31+
32+
@Override
33+
public ValidationResult onValidate(HollowProducer.ReadState readState) {
34+
ValidationResult.ValidationResultBuilder vrb = ValidationResult.from(this);
35+
36+
Integer minRecordCount = minRecordCountSupplier.get();
37+
// 1<<29 is the max no. of records supported in a Hollow type
38+
if (minRecordCount == null || minRecordCount < 0 || minRecordCount > (1<<29)) {
39+
String message = String.format(INVALID_THRESHOLD, typeName, minRecordCount);
40+
return vrb.failed(message);
41+
}
42+
43+
vrb.detail(DATA_TYPE_NAME, typeName)
44+
.detail(ALLOWABLE_MIN_RECORD_COUNT_NAME, minRecordCount);
45+
46+
HollowTypeReadState typeState = readState.getStateEngine().getTypeState(typeName);
47+
int recordCount = typeState.getPopulatedOrdinals().cardinality();
48+
49+
vrb.detail(RECORD_COUNT_NAME, recordCount);
50+
51+
if (recordCount < minRecordCount) {
52+
String message = String.format(FAILED_MIN_RECORD_COUNT_VALIDATION, typeName, recordCount, minRecordCount);
53+
return vrb.failed(message);
54+
}
55+
56+
return vrb.passed();
57+
}
58+
59+
}

‎hollow/src/test/java/com/netflix/hollow/api/producer/validation/MinimumRecordCountValidatorTest.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package com.netflix.hollow.api.producer.validation;
2+
3+
import static org.junit.Assert.assertEquals;
4+
import static org.junit.Assert.assertTrue;
5+
import static org.junit.Assert.fail;
6+
7+
import com.netflix.hollow.api.producer.HollowProducer;
8+
import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
9+
import com.netflix.hollow.test.InMemoryBlobStore;
10+
import java.util.function.Supplier;
11+
import org.junit.Before;
12+
import org.junit.Test;
13+
14+
public class MinimumRecordCountValidatorTest {
15+
16+
private InMemoryBlobStore blobStore;
17+
18+
@Before
19+
public void setUp() {
20+
blobStore = new InMemoryBlobStore();
21+
}
22+
23+
@Test
24+
public void testPassLessThanOrEqualToThresholdThreshold() {
25+
try {
26+
testHelper(1000000, () -> 500000);
27+
testHelper(10, () -> 10);
28+
} catch (Exception e) {
29+
fail(); //should not reach here
30+
}
31+
}
32+
33+
@Test
34+
public void testFailInvalidThreshold() {
35+
try {
36+
testHelper(5, () -> null);
37+
fail(); //should not reach here
38+
} catch (Exception e) {
39+
assertTrue(e instanceof ValidationStatusException);
40+
ValidationStatusException expected = (ValidationStatusException) e;
41+
assertEquals(1, expected.getValidationStatus().getResults().size());
42+
assertEquals("Minimum record count validation for type Integer has failed due to invalid threshold: null", expected.getValidationStatus().getResults().get(0).getMessage());
43+
}
44+
45+
try {
46+
testHelper(5, () -> -1);
47+
fail(); //should not reach here
48+
} catch (Exception e) {
49+
assertTrue(e instanceof ValidationStatusException);
50+
ValidationStatusException expected = (ValidationStatusException) e;
51+
assertEquals(1, expected.getValidationStatus().getResults().size());
52+
assertEquals("Minimum record count validation for type Integer has failed due to invalid threshold: -1", expected.getValidationStatus().getResults().get(0).getMessage());
53+
}
54+
55+
try {
56+
testHelper(5, () -> (1<<29) + 1);
57+
fail(); //should not reach here
58+
} catch (Exception e) {
59+
assertTrue(e instanceof ValidationStatusException);
60+
ValidationStatusException expected = (ValidationStatusException) e;
61+
assertEquals(1, expected.getValidationStatus().getResults().size());
62+
assertEquals("Minimum record count validation for type Integer has failed due to invalid threshold: 536870913", expected.getValidationStatus().getResults().get(0).getMessage());
63+
}
64+
}
65+
66+
@Test
67+
public void testFailThresholdBreached() {
68+
try {
69+
testHelper(5, () -> 6);
70+
fail();
71+
} catch (Exception e) {
72+
assertTrue(e instanceof ValidationStatusException);
73+
ValidationStatusException expected = (ValidationStatusException) e;
74+
assertEquals(1, expected.getValidationStatus().getResults().size());
75+
assertEquals("Minimum record count validation for type Integer has failed since the record count 5 is less than the configured threshold of 6", expected.getValidationStatus().getResults().get(0).getMessage());
76+
}
77+
}
78+
79+
private void testHelper(int currentRecordCount,
80+
Supplier<Integer> minRecordCountSupplier) {
81+
82+
HollowProducer producer = HollowProducer.withPublisher(blobStore)
83+
.withBlobStager(new HollowInMemoryBlobStager())
84+
.withListener(new MinimumRecordCountValidator("Integer", minRecordCountSupplier)).build();
85+
86+
producer.runCycle((state) -> {
87+
for (int i = 0; i < currentRecordCount; i++) {
88+
state.add(i);
89+
}
90+
});
91+
}
92+
}

‎hollow/src/test/java/com/netflix/hollow/api/producer/validation/RecordCountPercentChangeValidatorTests.java‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
package com.netflix.hollow.api.producer.validation;
22

3+
import static org.junit.Assert.assertEquals;
4+
import static org.junit.Assert.assertTrue;
5+
import static org.junit.Assert.fail;
6+
37
import com.netflix.hollow.api.producer.HollowProducer;
48
import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
59
import com.netflix.hollow.core.write.objectmapper.HollowPrimaryKey;
610
import com.netflix.hollow.test.InMemoryBlobStore;
7-
import org.junit.Assert;
8-
import org.junit.Before;
9-
import org.junit.Test;
10-
1111
import java.util.ArrayList;
1212
import java.util.List;
13+
import org.junit.Before;
14+
import org.junit.Test;
1315

1416
public class RecordCountPercentChangeValidatorTests {
1517
private InMemoryBlobStore blobStore;
@@ -26,7 +28,7 @@ public void testPassLessThanThreshold() {
2628
.withAddedPercentageThreshold(() -> 0.5f)
2729
.build(), 0, 0, 0);
2830
} catch (Exception e) {
29-
Assert.fail(); //should not reach here
31+
fail(); //should not reach here
3032
}
3133
}
3234

@@ -37,7 +39,7 @@ public void testPassThresholdNotSet() {
3739
.withAddedPercentageThreshold(() -> 0.5f)
3840
.build(), 0, 0, 3);
3941
} catch (Exception e) {
40-
Assert.fail(); //should not reach here
42+
fail(); //should not reach here
4143
}
4244
}
4345

@@ -47,9 +49,11 @@ public void testAddExceedThreshold() {
4749
testHelper(RecordCountPercentChangeValidator.Threshold.builder()
4850
.withAddedPercentageThreshold(() -> 0.5f)
4951
.build(), 0, 4, 0);
50-
Assert.fail();
51-
} catch (ValidationStatusException expected) {
52-
Assert.assertEquals(1, expected.getValidationStatus().getResults().size());
52+
fail();
53+
} catch (Exception e) {
54+
assertTrue(e instanceof ValidationStatusException);
55+
ValidationStatusException expected = (ValidationStatusException) e;
56+
assertEquals(1, expected.getValidationStatus().getResults().size());
5357
}
5458
}
5559

@@ -59,9 +63,11 @@ public void testRemoveExceedThreshold() {
5963
testHelper(RecordCountPercentChangeValidator.Threshold.builder()
6064
.withRemovedPercentageThreshold(() -> 0.5f)
6165
.build(), 0, 0, 4);
62-
Assert.fail();
63-
} catch (ValidationStatusException expected) {
64-
Assert.assertEquals(1, expected.getValidationStatus().getResults().size());
66+
fail();
67+
} catch (Exception e) {
68+
assertTrue(e instanceof ValidationStatusException);
69+
ValidationStatusException expected = (ValidationStatusException) e;
70+
assertEquals(1, expected.getValidationStatus().getResults().size());
6571
}
6672
}
6773

@@ -73,9 +79,11 @@ public void testUpdateExceedThreshold() {
7379
.withAddedPercentageThreshold(() -> 0.5f)
7480
.withAddedPercentageThreshold(() -> 0.5f)
7581
.build(), 4, 1, 1);
76-
Assert.fail();
77-
} catch (ValidationStatusException expected) {
78-
Assert.assertEquals(1, expected.getValidationStatus().getResults().size());
82+
fail();
83+
} catch (Exception e) {
84+
assertTrue(e instanceof ValidationStatusException);
85+
ValidationStatusException expected = (ValidationStatusException) e;
86+
assertEquals(1, expected.getValidationStatus().getResults().size());
7987
}
8088
}
8189

0 commit comments

Comments
 (0)