Skip to content

Commit ab75f80

Browse files
committed
MLH-1240 revert all optimisations and keep current cinv based full refresh
1 parent adff2bb commit ab75f80

File tree

9 files changed

+348
-464
lines changed

9 files changed

+348
-464
lines changed

common/src/main/java/org/apache/atlas/repository/Constants.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -538,13 +538,6 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV }
538538
add(STAKEHOLDER_TITLE_ENTITY_TYPE);
539539
}};
540540

541-
public static final String TYPEDEF_ENUM_CACHE_LATEST_VERSION = "typdef.enum.cache.version";
542-
public static final String TYPEDEF_BUSINESS_METADATA_CACHE_LATEST_VERSION = "typdef.bm.cache.version";
543-
public static final String TYPEDEF_CLASSIFICATION_METADATA_CACHE_LATEST_VERSION = "typdef.cls.cache.version";
544-
public static final String TYPEDEF_STRUCT_CACHE_LATEST_VERSION = "typdef.struct.cache.version";
545-
public static final String TYPEDEF_ENTITY_CACHE_LATEST_VERSION = "typdef.entity.cache.version";
546-
public static final String TYPEDEF_RELATIONSHIP_CACHE_LATEST_VERSION = "typdef.relationship.cache.version";
547-
548541
private Constants() {
549542
}
550543

common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import org.apache.atlas.service.metrics.MetricUtils;
66
import org.apache.commons.configuration.Configuration;
77
import org.apache.commons.lang.ArrayUtils;
8-
import org.apache.commons.lang.StringUtils;
98
import org.redisson.api.RLock;
109
import org.redisson.api.RedissonClient;
1110
import org.redisson.config.Config;
@@ -192,22 +191,6 @@ public String getValue(String key) {
192191
}
193192
}
194193

195-
@Override
196-
public String getValue(String key, String defaultValue) {
197-
try {
198-
String value = getValue(key);
199-
if (StringUtils.isEmpty(value)) {
200-
return defaultValue;
201-
} else {
202-
return value;
203-
}
204-
} catch (Exception e) {
205-
MetricUtils.recordRedisConnectionFailure();
206-
getLogger().error("Redis getValue operation failed for key: {}", key, e);
207-
throw e;
208-
}
209-
}
210-
211194
@Override
212195
public String putValue(String key, String value) {
213196
try {

common/src/main/java/org/apache/atlas/service/redis/RedisService.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ public interface RedisService {
1616

1717
String getValue(String key);
1818

19-
String getValue(String key, String defaultValue);
20-
2119
String putValue(String key, String value);
2220

2321
String putValue(String key, String value, int timeout);

repository/src/main/java/org/apache/atlas/repository/graph/TypeCacheRefresher.java

Lines changed: 156 additions & 167 deletions
Large diffs are not rendered by default.

repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java

Lines changed: 1 addition & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
import org.apache.atlas.repository.patches.SuperTypesUpdatePatch;
5151
import org.apache.atlas.repository.patches.AtlasPatchManager;
5252
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
53-
import org.apache.atlas.service.redis.RedisService;
5453
import org.apache.atlas.store.AtlasTypeDefStore;
5554
import org.apache.atlas.type.AtlasEntityType;
5655
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
@@ -102,23 +101,15 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
102101
private final Configuration conf;
103102
private final AtlasGraph graph;
104103
private final AtlasPatchManager patchManager;
105-
private final RedisService redisService;
106-
private static long CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION;
107-
private static long CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION;
108-
private static long CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION;
109-
private static long CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION;
110-
private static long CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION;
111-
private static long CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION;
112104

113105
@Inject
114106
public AtlasTypeDefStoreInitializer(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
115-
AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager, RedisService redisService) throws AtlasBaseException {
107+
AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager) throws AtlasBaseException {
116108
this.typeDefStore = typeDefStore;
117109
this.typeRegistry = typeRegistry;
118110
this.conf = conf;
119111
this.graph = graph;
120112
this.patchManager = patchManager;
121-
this.redisService = redisService;
122113
}
123114

124115
@PostConstruct
@@ -127,12 +118,6 @@ public void init() {
127118

128119
if (!HAConfiguration.isHAEnabled(conf)) {
129120
startInternal();
130-
CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_ENUM_CACHE_LATEST_VERSION, "1"));
131-
CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_BUSINESS_METADATA_CACHE_LATEST_VERSION, "1"));
132-
CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_CLASSIFICATION_METADATA_CACHE_LATEST_VERSION, "1"));
133-
CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_STRUCT_CACHE_LATEST_VERSION, "1"));
134-
CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_ENTITY_CACHE_LATEST_VERSION, "1"));
135-
CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_RELATIONSHIP_CACHE_LATEST_VERSION, "1"));
136121
} else {
137122
LOG.info("AtlasTypeDefStoreInitializer.init(): deferring type loading until instance activation");
138123
}
@@ -422,54 +407,6 @@ public int getHandlerOrder() {
422407
return HandlerOrder.TYPEDEF_STORE_INITIALIZER.getOrder();
423408
}
424409

425-
public static long getCurrentEnumTypedefInternalVersion() {
426-
return CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION;
427-
}
428-
429-
public static void setCurrentEnumTypedefInternalVersion(long version) {
430-
CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION = version;
431-
}
432-
433-
public static long getCurrentBMTypedefInternalVersion() {
434-
return CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION;
435-
}
436-
437-
public static void setCurrentBMTypedefInternalVersion(long version) {
438-
CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION = version;
439-
}
440-
441-
public static long getCurrentClassificationTypedefInternalVersion() {
442-
return CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION;
443-
}
444-
445-
public static void setCurrentClassificationTypedefInternalVersion(long version) {
446-
CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION = version;
447-
}
448-
449-
public static long getCurrentStructTypedefInternalVersion() {
450-
return CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION;
451-
}
452-
453-
public static void setCurrentStructTypedefInternalVersion(long version) {
454-
CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION = version;
455-
}
456-
457-
public static long getCurrentEntityTypedefInternalVersion() {
458-
return CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION;
459-
}
460-
461-
public static void setCurrentEntityTypedefInternalVersion(long version) {
462-
CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION = version;
463-
}
464-
465-
public static long getCurrentRelationshipTypedefInternalVersion() {
466-
return CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION;
467-
}
468-
469-
public static void setCurrentRelationshipTypedefInternalVersion(long version) {
470-
CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION = version;
471-
}
472-
473410
private static boolean updateTypeAttributes(AtlasStructDef oldStructDef, AtlasStructDef newStructDef, boolean checkTypeVersion) {
474411
boolean ret = isTypeUpdateApplicable(oldStructDef, newStructDef, checkTypeVersion);
475412

repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java

Lines changed: 39 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -123,135 +123,17 @@ public void init() throws AtlasBaseException {
123123
}
124124

125125
@Override
126-
public void reloadEnumTypeDefs() throws AtlasBaseException {
127-
LOG.info("==> AtlasTypeDefGraphStore.reloadEnumTypeDefs()");
128-
AtlasTransientTypeRegistry ttr = null;
129-
boolean commitUpdates = false;
130-
131-
try {
132-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
133-
List<AtlasEnumDef> enumDefs = getEnumDefStore(ttr).getAll();
134-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllEnumDefs()) {
135-
if (atlasBaseTypeDef instanceof AtlasEnumDef) {
136-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
137-
}
138-
}
139-
ttr.addTypes(enumDefs);
140-
commitUpdates = true;
141-
} finally {
142-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
143-
LOG.info("<== AtlasTypeDefGraphStore.reloadEnumTypeDefs()");
144-
}
145-
}
146-
147-
@Override
148-
public void reloadStructTypeDefs() throws AtlasBaseException {
149-
LOG.info("==> AtlasTypeDefGraphStore.reloadStructTypeDefs()");
150-
AtlasTransientTypeRegistry ttr = null;
151-
boolean commitUpdates = false;
152-
153-
try {
154-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
155-
List<AtlasStructDef> structDefs = getStructDefStore(ttr).getAll();
156-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllStructDefs()) {
157-
if (atlasBaseTypeDef instanceof AtlasStructDef) {
158-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
159-
}
160-
}
161-
ttr.addTypes(structDefs);
162-
commitUpdates = true;
163-
} finally {
164-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
165-
LOG.info("<== AtlasTypeDefGraphStore.reloadStructTypeDefs()");
166-
}
167-
}
168-
169-
@Override
170-
public void reloadEntityTypeDefs() throws AtlasBaseException {
171-
LOG.info("==> AtlasTypeDefGraphStore.reloadEntityTypeDefs()");
172-
AtlasTransientTypeRegistry ttr = null;
173-
boolean commitUpdates = false;
174-
175-
try {
176-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
177-
List<AtlasEntityDef> entityDefs = getEntityDefStore(ttr).getAll();
178-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllEntityDefs()) {
179-
if (atlasBaseTypeDef instanceof AtlasEntityDef) {
180-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
181-
}
182-
}
183-
ttr.addTypes(entityDefs);
184-
commitUpdates = true;
185-
} finally {
186-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
187-
LOG.info("<== AtlasTypeDefGraphStore.reloadEntityTypeDefs()");
188-
}
189-
}
190-
191-
@Override
192-
public void reloadRelationshipTypeDefs() throws AtlasBaseException {
193-
LOG.info("==> AtlasTypeDefGraphStore.reloadRelationshipTypeDefs()");
194-
AtlasTransientTypeRegistry ttr = null;
195-
boolean commitUpdates = false;
196-
197-
try {
198-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
199-
List<AtlasRelationshipDef> relationshipDefs = getRelationshipDefStore(ttr).getAll();
200-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllRelationshipDefs()) {
201-
if (atlasBaseTypeDef instanceof AtlasRelationshipDef) {
202-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
203-
}
204-
}
205-
ttr.addTypes(relationshipDefs);
206-
commitUpdates = true;
207-
} finally {
208-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
209-
LOG.info("<== AtlasTypeDefGraphStore.reloadRelationshipTypeDefs()");
210-
}
211-
}
212-
213-
@Override
214-
public void reloadBusinessMetadataTypeDefs() throws AtlasBaseException {
215-
LOG.info("==> AtlasTypeDefGraphStore.reloadBusinessMetadataTypeDefs()");
216-
AtlasTransientTypeRegistry ttr = null;
217-
boolean commitUpdates = false;
218-
219-
try {
220-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
221-
List<AtlasBusinessMetadataDef> businessMetadataDefs = getBusinessMetadataDefStore(ttr).getAll();
222-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllBusinessMetadataDefs()) {
223-
if (atlasBaseTypeDef instanceof AtlasBusinessMetadataDef) {
224-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
225-
}
226-
}
227-
ttr.addTypes(businessMetadataDefs);
228-
commitUpdates = true;
229-
} finally {
230-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
231-
LOG.info("<== AtlasTypeDefGraphStore.reloadBusinessMetadataTypeDefs()");
232-
}
233-
}
234-
235-
@Override
236-
public void reloadClassificationMetadataTypeDefs() throws AtlasBaseException {
237-
LOG.info("==> AtlasTypeDefGraphStore.reloadClassificationMetadataTypeDefs()");
238-
AtlasTransientTypeRegistry ttr = null;
239-
boolean commitUpdates = false;
240-
241-
try {
242-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
243-
List<AtlasClassificationDef> classificationDefs = getClassificationDefStore(ttr).getAll();
244-
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllClassificationDefs()) {
245-
if (atlasBaseTypeDef instanceof AtlasClassificationDef) {
246-
ttr.removeTypeByName(atlasBaseTypeDef.getName());
247-
}
248-
}
249-
ttr.addTypes(classificationDefs);
250-
commitUpdates = true;
251-
} finally {
252-
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
253-
LOG.info("<== AtlasTypeDefGraphStore.reloadClassificationMetadataTypeDefs()");
254-
}
126+
public void initWithoutLock() throws AtlasBaseException {
127+
// need even better approach than this
128+
AtlasTypesDef typesDef = new AtlasTypesDef(getEnumDefStore(typeRegistry).getAll(),
129+
getStructDefStore(typeRegistry).getAll(),
130+
getClassificationDefStore(typeRegistry).getAll(),
131+
getEntityDefStore(typeRegistry).getAll(),
132+
getRelationshipDefStore(typeRegistry).getAll(),
133+
getBusinessMetadataDefStore(typeRegistry).getAll());
134+
135+
rectifyTypeErrorsIfAny(typesDef);
136+
typeRegistry.addTypes(typesDef);
255137
}
256138

257139
@Override
@@ -831,7 +713,7 @@ public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
831713

832714
@Override
833715
@GraphTransaction
834-
public AtlasBaseTypeDef deleteTypeByName(String typeName) throws AtlasBaseException {
716+
public void deleteTypeByName(String typeName) throws AtlasBaseException {
835717
AtlasType atlasType = typeRegistry.getType(typeName);
836718
if (atlasType == null) {
837719
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS.TYPE_NAME_NOT_FOUND, typeName);
@@ -855,58 +737,52 @@ public AtlasBaseTypeDef deleteTypeByName(String typeName) throws AtlasBaseExcept
855737
}
856738

857739
deleteTypesDef(typesDef);
858-
return baseTypeDef;
859740
}
860741

861742
@Override
862743
public AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException {
863744
final AtlasTypesDef typesDef = new AtlasTypesDef();
864745
Predicate searchPredicates = FilterUtil.getPredicateFromSearchFilter(searchFilter);
865-
AtlasTransientTypeRegistry ttr = null;
866-
try {
867-
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
868-
for (AtlasEnumType enumType : ttr.getAllEnumTypes()) {
869-
if (searchPredicates.evaluate(enumType)) {
870-
typesDef.getEnumDefs().add(enumType.getEnumDef());
871-
}
746+
747+
for(AtlasEnumType enumType : typeRegistry.getAllEnumTypes()) {
748+
if (searchPredicates.evaluate(enumType)) {
749+
typesDef.getEnumDefs().add(enumType.getEnumDef());
872750
}
751+
}
873752

874-
for (AtlasStructType structType : ttr.getAllStructTypes()) {
875-
if (searchPredicates.evaluate(structType)) {
876-
typesDef.getStructDefs().add(structType.getStructDef());
877-
}
753+
for(AtlasStructType structType : typeRegistry.getAllStructTypes()) {
754+
if (searchPredicates.evaluate(structType)) {
755+
typesDef.getStructDefs().add(structType.getStructDef());
878756
}
757+
}
879758

880-
for (AtlasClassificationType classificationType : ttr.getAllClassificationTypes()) {
881-
if (searchPredicates.evaluate(classificationType)) {
882-
typesDef.getClassificationDefs().add(classificationType.getClassificationDef());
883-
}
759+
for(AtlasClassificationType classificationType : typeRegistry.getAllClassificationTypes()) {
760+
if (searchPredicates.evaluate(classificationType)) {
761+
typesDef.getClassificationDefs().add(classificationType.getClassificationDef());
884762
}
763+
}
885764

886-
for (AtlasEntityType entityType : ttr.getAllEntityTypes()) {
887-
if (searchPredicates.evaluate(entityType)) {
888-
typesDef.getEntityDefs().add(entityType.getEntityDef());
889-
}
765+
for(AtlasEntityType entityType : typeRegistry.getAllEntityTypes()) {
766+
if (searchPredicates.evaluate(entityType)) {
767+
typesDef.getEntityDefs().add(entityType.getEntityDef());
890768
}
769+
}
891770

892-
for (AtlasRelationshipType relationshipType : ttr.getAllRelationshipTypes()) {
893-
if (searchPredicates.evaluate(relationshipType)) {
894-
typesDef.getRelationshipDefs().add(relationshipType.getRelationshipDef());
895-
}
771+
for(AtlasRelationshipType relationshipType : typeRegistry.getAllRelationshipTypes()) {
772+
if (searchPredicates.evaluate(relationshipType)) {
773+
typesDef.getRelationshipDefs().add(relationshipType.getRelationshipDef());
896774
}
775+
}
897776

898-
for (AtlasBusinessMetadataType businessMetadataType : ttr.getAllBusinessMetadataTypes()) {
899-
if (searchPredicates.evaluate(businessMetadataType)) {
900-
typesDef.getBusinessMetadataDefs().add(businessMetadataType.getBusinessMetadataDef());
901-
}
777+
for(AtlasBusinessMetadataType businessMetadataType : typeRegistry.getAllBusinessMetadataTypes()) {
778+
if (searchPredicates.evaluate(businessMetadataType)) {
779+
typesDef.getBusinessMetadataDefs().add(businessMetadataType.getBusinessMetadataDef());
902780
}
781+
}
903782

904-
AtlasAuthorizationUtils.filterTypesDef(new AtlasTypesDefFilterRequest(typesDef));
783+
AtlasAuthorizationUtils.filterTypesDef(new AtlasTypesDefFilterRequest(typesDef));
905784

906-
return typesDef;
907-
} finally {
908-
typeRegistry.releaseTypeRegistryForUpdate(ttr, false);
909-
}
785+
return typesDef;
910786
}
911787

912788
@Override

0 commit comments

Comments
 (0)