Skip to content

Staging helper #5428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 23, 2025
Merged

Staging helper #5428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ab75f80
MLH-1240 revert all optimisations and keep current cinv based full re…
sriram-atlan Sep 19, 2025
478882b
MLH-1240 branch to build
sriram-atlan Sep 19, 2025
945da59
MLH-1240 revert initWithoutlock method too
sriram-atlan Sep 19, 2025
3ba9eca
Merge pull request #5408 from atlanhq/revert-custom-metadata-changes
sriram-atlan Sep 20, 2025
9002aa2
Merge branch 'master' into staging-helper
nikhilbonte21 Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions common/src/main/java/org/apache/atlas/repository/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,13 +536,6 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV }
add(STAKEHOLDER_TITLE_ENTITY_TYPE);
}};

public static final String TYPEDEF_ENUM_CACHE_LATEST_VERSION = "typdef.enum.cache.version";
public static final String TYPEDEF_BUSINESS_METADATA_CACHE_LATEST_VERSION = "typdef.bm.cache.version";
public static final String TYPEDEF_CLASSIFICATION_METADATA_CACHE_LATEST_VERSION = "typdef.cls.cache.version";
public static final String TYPEDEF_STRUCT_CACHE_LATEST_VERSION = "typdef.struct.cache.version";
public static final String TYPEDEF_ENTITY_CACHE_LATEST_VERSION = "typdef.entity.cache.version";
public static final String TYPEDEF_RELATIONSHIP_CACHE_LATEST_VERSION = "typdef.relationship.cache.version";

private Constants() {
}

Expand Down
17 changes: 0 additions & 17 deletions common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import org.apache.atlas.service.metrics.MetricUtils;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.redisson.config.Config;
Expand Down Expand Up @@ -192,22 +191,6 @@ public String getValue(String key) {
}
}

@Override
public String getValue(String key, String defaultValue) {
try {
String value = getValue(key);
if (StringUtils.isEmpty(value)) {
return defaultValue;
} else {
return value;
}
} catch (Exception e) {
MetricUtils.recordRedisConnectionFailure();
getLogger().error("Redis getValue operation failed for key: {}", key, e);
throw e;
}
}

@Override
public String putValue(String key, String value) {
try {
Expand Down
2 changes: 0 additions & 2 deletions common/src/main/java/org/apache/atlas/service/redis/RedisService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ public interface RedisService {

String getValue(String key);

String getValue(String key, String defaultValue);

String putValue(String key, String value);

String putValue(String key, String value, int timeout);
Expand Down
323 changes: 156 additions & 167 deletions repository/src/main/java/org/apache/atlas/repository/graph/TypeCacheRefresher.java
View file
Open in desktop

Large diffs are not rendered by default.

65 changes: 1 addition & 64 deletions ...c/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
import org.apache.atlas.repository.patches.SuperTypesUpdatePatch;
import org.apache.atlas.repository.patches.AtlasPatchManager;
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.service.redis.RedisService;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
Expand Down Expand Up @@ -102,23 +101,15 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private final Configuration conf;
private final AtlasGraph graph;
private final AtlasPatchManager patchManager;
private final RedisService redisService;
private static long CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION;
private static long CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION;
private static long CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION;
private static long CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION;
private static long CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION;
private static long CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION;

@Inject
public AtlasTypeDefStoreInitializer(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager, RedisService redisService) throws AtlasBaseException {
AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager) throws AtlasBaseException {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.conf = conf;
this.graph = graph;
this.patchManager = patchManager;
this.redisService = redisService;
}

@PostConstruct
Expand All @@ -127,12 +118,6 @@ public void init() {

if (!HAConfiguration.isHAEnabled(conf)) {
startInternal();
CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_ENUM_CACHE_LATEST_VERSION, "1"));
CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_BUSINESS_METADATA_CACHE_LATEST_VERSION, "1"));
CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_CLASSIFICATION_METADATA_CACHE_LATEST_VERSION, "1"));
CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_STRUCT_CACHE_LATEST_VERSION, "1"));
CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_ENTITY_CACHE_LATEST_VERSION, "1"));
CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION = Long.parseLong(redisService.getValue(Constants.TYPEDEF_RELATIONSHIP_CACHE_LATEST_VERSION, "1"));
} else {
LOG.info("AtlasTypeDefStoreInitializer.init(): deferring type loading until instance activation");
}
Expand Down Expand Up @@ -422,54 +407,6 @@ public int getHandlerOrder() {
return HandlerOrder.TYPEDEF_STORE_INITIALIZER.getOrder();
}

public static long getCurrentEnumTypedefInternalVersion() {
return CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentEnumTypedefInternalVersion(long version) {
CURRENT_ENUM_TYPEDEF_INTERNAL_VERSION = version;
}

public static long getCurrentBMTypedefInternalVersion() {
return CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentBMTypedefInternalVersion(long version) {
CURRENT_BUSINESS_METADATA_TYPEDEF_INTERNAL_VERSION = version;
}

public static long getCurrentClassificationTypedefInternalVersion() {
return CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentClassificationTypedefInternalVersion(long version) {
CURRENT_CLASSIFICATION_TYPEDEF_INTERNAL_VERSION = version;
}

public static long getCurrentStructTypedefInternalVersion() {
return CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentStructTypedefInternalVersion(long version) {
CURRENT_STRUCT_TYPEDEF_INTERNAL_VERSION = version;
}

public static long getCurrentEntityTypedefInternalVersion() {
return CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentEntityTypedefInternalVersion(long version) {
CURRENT_ENTITY_TYPEDEF_INTERNAL_VERSION = version;
}

public static long getCurrentRelationshipTypedefInternalVersion() {
return CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION;
}

public static void setCurrentRelationshipTypedefInternalVersion(long version) {
CURRENT_RELATIONSHIP_TYPEDEF_INTERNAL_VERSION = version;
}

private static boolean updateTypeAttributes(AtlasStructDef oldStructDef, AtlasStructDef newStructDef, boolean checkTypeVersion) {
boolean ret = isTypeUpdateApplicable(oldStructDef, newStructDef, checkTypeVersion);

Expand Down
194 changes: 28 additions & 166 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,138 +122,6 @@ public void init() throws AtlasBaseException {
}
}

@Override
public void reloadEnumTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadEnumTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasEnumDef> enumDefs = getEnumDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllEnumDefs()) {
if (atlasBaseTypeDef instanceof AtlasEnumDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(enumDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadEnumTypeDefs()");
}
}

@Override
public void reloadStructTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadStructTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasStructDef> structDefs = getStructDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllStructDefs()) {
if (atlasBaseTypeDef instanceof AtlasStructDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(structDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadStructTypeDefs()");
}
}

@Override
public void reloadEntityTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadEntityTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasEntityDef> entityDefs = getEntityDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllEntityDefs()) {
if (atlasBaseTypeDef instanceof AtlasEntityDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(entityDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadEntityTypeDefs()");
}
}

@Override
public void reloadRelationshipTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadRelationshipTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasRelationshipDef> relationshipDefs = getRelationshipDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllRelationshipDefs()) {
if (atlasBaseTypeDef instanceof AtlasRelationshipDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(relationshipDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadRelationshipTypeDefs()");
}
}

@Override
public void reloadBusinessMetadataTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadBusinessMetadataTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasBusinessMetadataDef> businessMetadataDefs = getBusinessMetadataDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllBusinessMetadataDefs()) {
if (atlasBaseTypeDef instanceof AtlasBusinessMetadataDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(businessMetadataDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadBusinessMetadataTypeDefs()");
}
}

@Override
public void reloadClassificationMetadataTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.reloadClassificationMetadataTypeDefs()");
AtlasTransientTypeRegistry ttr = null;
boolean commitUpdates = false;

try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
List<AtlasClassificationDef> classificationDefs = getClassificationDefStore(ttr).getAll();
for (AtlasBaseTypeDef atlasBaseTypeDef : ttr.getAllClassificationDefs()) {
if (atlasBaseTypeDef instanceof AtlasClassificationDef) {
ttr.removeTypeByName(atlasBaseTypeDef.getName());
}
}
ttr.addTypes(classificationDefs);
commitUpdates = true;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, commitUpdates);
LOG.info("<== AtlasTypeDefGraphStore.reloadClassificationMetadataTypeDefs()");
}
}

@Override
public AtlasEnumDef getEnumDefByName(String name) throws AtlasBaseException {
AtlasEnumDef ret = typeRegistry.getEnumDefByName(name);
Expand Down Expand Up @@ -831,7 +699,7 @@ public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {

@Override
@GraphTransaction
public AtlasBaseTypeDef deleteTypeByName(String typeName) throws AtlasBaseException {
public void deleteTypeByName(String typeName) throws AtlasBaseException {
AtlasType atlasType = typeRegistry.getType(typeName);
if (atlasType == null) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS.TYPE_NAME_NOT_FOUND, typeName);
Expand All @@ -855,58 +723,52 @@ public AtlasBaseTypeDef deleteTypeByName(String typeName) throws AtlasBaseExcept
}

deleteTypesDef(typesDef);
return baseTypeDef;
}

@Override
public AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException {
final AtlasTypesDef typesDef = new AtlasTypesDef();
Predicate searchPredicates = FilterUtil.getPredicateFromSearchFilter(searchFilter);
AtlasTransientTypeRegistry ttr = null;
try {
ttr = typeRegistry.lockTypeRegistryForUpdate(5);
for (AtlasEnumType enumType : ttr.getAllEnumTypes()) {
if (searchPredicates.evaluate(enumType)) {
typesDef.getEnumDefs().add(enumType.getEnumDef());
}

for(AtlasEnumType enumType : typeRegistry.getAllEnumTypes()) {
if (searchPredicates.evaluate(enumType)) {
typesDef.getEnumDefs().add(enumType.getEnumDef());
}
}

for (AtlasStructType structType : ttr.getAllStructTypes()) {
if (searchPredicates.evaluate(structType)) {
typesDef.getStructDefs().add(structType.getStructDef());
}
for(AtlasStructType structType : typeRegistry.getAllStructTypes()) {
if (searchPredicates.evaluate(structType)) {
typesDef.getStructDefs().add(structType.getStructDef());
}
}

for (AtlasClassificationType classificationType : ttr.getAllClassificationTypes()) {
if (searchPredicates.evaluate(classificationType)) {
typesDef.getClassificationDefs().add(classificationType.getClassificationDef());
}
for(AtlasClassificationType classificationType : typeRegistry.getAllClassificationTypes()) {
if (searchPredicates.evaluate(classificationType)) {
typesDef.getClassificationDefs().add(classificationType.getClassificationDef());
}
}

for (AtlasEntityType entityType : ttr.getAllEntityTypes()) {
if (searchPredicates.evaluate(entityType)) {
typesDef.getEntityDefs().add(entityType.getEntityDef());
}
for(AtlasEntityType entityType : typeRegistry.getAllEntityTypes()) {
if (searchPredicates.evaluate(entityType)) {
typesDef.getEntityDefs().add(entityType.getEntityDef());
}
}

for (AtlasRelationshipType relationshipType : ttr.getAllRelationshipTypes()) {
if (searchPredicates.evaluate(relationshipType)) {
typesDef.getRelationshipDefs().add(relationshipType.getRelationshipDef());
}
for(AtlasRelationshipType relationshipType : typeRegistry.getAllRelationshipTypes()) {
if (searchPredicates.evaluate(relationshipType)) {
typesDef.getRelationshipDefs().add(relationshipType.getRelationshipDef());
}
}

for (AtlasBusinessMetadataType businessMetadataType : ttr.getAllBusinessMetadataTypes()) {
if (searchPredicates.evaluate(businessMetadataType)) {
typesDef.getBusinessMetadataDefs().add(businessMetadataType.getBusinessMetadataDef());
}
for(AtlasBusinessMetadataType businessMetadataType : typeRegistry.getAllBusinessMetadataTypes()) {
if (searchPredicates.evaluate(businessMetadataType)) {
typesDef.getBusinessMetadataDefs().add(businessMetadataType.getBusinessMetadataDef());
}
}

AtlasAuthorizationUtils.filterTypesDef(new AtlasTypesDefFilterRequest(typesDef));
AtlasAuthorizationUtils.filterTypesDef(new AtlasTypesDefFilterRequest(typesDef));

return typesDef;
} finally {
typeRegistry.releaseTypeRegistryForUpdate(ttr, false);
}
return typesDef;
}

@Override
Expand Down
Loading
Loading