Skip to content

MESH-661 : Value limit for keyword property #5439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 24, 2025
Merged

MESH-661 : Value limit for keyword property #5439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions ...main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.store.graph.v2;

import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasTypeAccessRequest;
Expand Down Expand Up @@ -52,6 +53,9 @@ public class AtlasBusinessMetadataDefStoreV2 extends AtlasAbstractDefStoreV2<Atl

private final EntityDiscoveryService entityDiscoveryService;

private static final int DEFAULT_RICH_TEXT_ATTRIBUTE_LIMIT = 15;
private static final String RICH_TEXT_ATTRIBUTE_LIMIT_PROPERTY = "atlas.business.metadata.richtext.limit";

@Inject
public AtlasBusinessMetadataDefStoreV2(AtlasTypeDefGraphStoreV2 typeDefStore, AtlasTypeRegistry typeRegistry, EntityDiscoveryService entityDiscoveryService) {
super(typeDefStore, typeRegistry);
Expand Down Expand Up @@ -101,6 +105,15 @@ public AtlasVertex preCreate(AtlasBusinessMetadataDef businessMetadataDef) throw
return ret;
}

private int getRichTextAttributeLimit() {
try {
return ApplicationProperties.get().getInt(RICH_TEXT_ATTRIBUTE_LIMIT_PROPERTY, DEFAULT_RICH_TEXT_ATTRIBUTE_LIMIT);
} catch (Exception e) {
LOG.warn("Failed to read rich text attribute limit configuration, using default: {}", DEFAULT_RICH_TEXT_ATTRIBUTE_LIMIT);
return DEFAULT_RICH_TEXT_ATTRIBUTE_LIMIT;
}
}

@Override
public void validateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
super.validateType(typeDef);
Expand Down Expand Up @@ -463,11 +476,13 @@ private void validateRichTextAttributeLimit(AtlasBusinessMetadataDef businessMet
int existingRichTextCount = countExistingRichTextAttributes(businessMetadataDef.getGuid());
int totalRichTextCount = existingRichTextCount + newRichTextCount;

if (totalRichTextCount > 2) {
int limit = getRichTextAttributeLimit();

if (totalRichTextCount > limit) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS,
String.format("Cannot create business metadata attributes. Total rich text attributes would exceed limit of 15. " +
"Current: %d, Attempting to add: %d, Limit: 15",
existingRichTextCount, newRichTextCount));
String.format("Cannot create business metadata attributes. Total rich text attributes would exceed limit of %d. " +
"Current: %d, Attempting to add: %d, Limit: %d",
existingRichTextCount, newRichTextCount, limit, limit));
Copy link

@cursor cursor bot Sep 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug

The String.format call for the rich text attribute limit error message has its arguments in the wrong order. This leads to incorrect values being displayed for the limit, current count, and attempted count in the error message.

Fix in Cursor Fix in Web

}
}

Expand Down
53 changes: 43 additions & 10 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,19 +63,10 @@
import org.apache.atlas.repository.util.TagDeNormAttributesUtil;
import org.apache.atlas.service.FeatureFlagStore;
import org.apache.atlas.tasks.TaskManagement;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasBusinessMetadataType.AtlasBusinessAttribute;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.AtlasPerfMetrics;
Expand Down Expand Up @@ -171,6 +162,8 @@ public class EntityGraphMapper {
private static final String ATTR_MEANINGS = "meanings";
private static final String ATTR_ANCHOR = "anchor";
private static final String ATTR_CATEGORIES = "categories";
private static final int ELASTICSEARCH_KEYWORD_MAX_BYTES = 32766;
private static final String UTF8_CHARSET = "UTF-8";
private static final List<String> ALLOWED_DATATYPES_FOR_DEFAULT_NULL = new ArrayList() {
{
add("int");
Expand Down Expand Up @@ -2459,6 +2452,44 @@ private static void validateCustomRelationshipCount(long size, AtlasVertex verte
}
}

private void validateElasticsearchKeywordSize(AtlasBusinessAttribute bmAttribute, Object attrValue, String fieldName, List<String> messages) {
if (!isKeywordMappedAttribute(bmAttribute)) {
return;
}

if (attrValue == null) {
return;
}

try {
String valueAsString = attrValue.toString();
byte[] valueBytes = valueAsString.getBytes(UTF8_CHARSET);

if (valueBytes.length > ELASTICSEARCH_KEYWORD_MAX_BYTES) {
messages.add(String.format("%s: Business attribute value exceeds Elasticsearch keyword limit. " +
"Size: %d bytes, Limit: %d bytes (32KB). Consider using a different field type or reducing the value size.",
fieldName, valueBytes.length, ELASTICSEARCH_KEYWORD_MAX_BYTES));
}
} catch (Exception e) {
LOG.warn("Failed to validate Elasticsearch keyword size for field: {}", fieldName, e);
}
}

private boolean isKeywordMappedAttribute(AtlasBusinessAttribute bmAttribute) {
AtlasAttributeDef attributeDef = bmAttribute.getAttributeDef();
AtlasType attrType = bmAttribute.getAttributeType();

if (attrType instanceof AtlasBuiltInTypes.AtlasStringType || attrType instanceof AtlasEnumType) {
return true;
}

if (attributeDef.getIndexType() == AtlasAttributeDef.IndexType.STRING) {
return true;
}

return false;
}

private void addInternalProductAttr(AttributeMutationContext ctx, List<Object> createdElements, List<AtlasEdge> deletedElements, List<Object> currentElements) throws AtlasBaseException {
MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend");
AtlasVertex toVertex = ctx.getReferringVertex();
Expand Down Expand Up @@ -5763,6 +5794,8 @@ private void validateBusinessAttributes(AtlasVertex entityVertex, AtlasEntityTyp
messages.add(fieldName + ": Business attribute-value exceeds maximum length limit");
}

validateElasticsearchKeywordSize(bmAttribute, attrValue, fieldName, messages);

} else if (!bmAttribute.getAttributeDef().getIsOptional()) {
final boolean isAttrValuePresent;

Expand Down
Loading