Skip to content

Commit 8698f76

Browse files
yingjianwu98Yingjian Wu
yingjianwu98
and
Yingjian Wu
authored
select definitionMetada with update (#649)
* select definitionMetada with update * remove readOnly=true * wip * wip * wip * wip * wip * wip * wip --------- Co-authored-by: Yingjian Wu <[email protected]>
1 parent c66198e commit 8698f76

File tree

9 files changed

+114
-22
lines changed

9 files changed

+114
-22
lines changed

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,5 +649,13 @@ public interface Config {
649649
* @return parentChildRelationshipProperties
650650
*/
651651
ParentChildRelationshipProperties getParentChildRelationshipProperties();
652+
653+
654+
/**
655+
* Whether we enable select for update during saveDefinitionMetadata.
656+
*
657+
* @return True if it should be.
658+
*/
659+
boolean isDefinitionMetadataSelectForUpdateEnabled();
652660
}
653661

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -708,4 +708,9 @@ public boolean isParentChildDropEnabled() {
708708
public ParentChildRelationshipProperties getParentChildRelationshipProperties() {
709709
return this.metacatProperties.getParentChildRelationshipProperties();
710710
}
711+
712+
@Override
713+
public boolean isDefinitionMetadataSelectForUpdateEnabled() {
714+
return this.metacatProperties.getUsermetadata().isDefinitionMetadataSelectForUpdateEnabled();
715+
}
711716
}

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ public class ServiceProperties {
4141
private int listTableNamesPageSize = 10000;
4242
private int listDatabaseEntitiesPageSize = 1000;
4343
private int listDatabaseNamesPageSize = 10000;
44-
4544
/**
4645
* Max related properties.
4746
*

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public class UserMetadata {
3232
private Config config = new Config();
3333
private int queryTimeoutInSeconds = 60;
3434
private int longQueryTimeoutInSeconds = 120;
35+
private boolean definitionMetadataSelectForUpdateEnabled;
3536

3637
/**
3738
* config related properties.

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/MetadataPreMergeInterceptor.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.netflix.metacat.common.server.usermetadata;
2+
3+
import com.fasterxml.jackson.databind.node.ObjectNode;
4+
import com.netflix.metacat.common.QualifiedName;
5+
import com.netflix.metacat.common.server.connectors.exception.InvalidMetadataException;
6+
7+
import java.util.Optional;
8+
9+
/**
10+
* MetadataPreMergeInterceptor: This interceptor runs before merging the existing metadata and the new metadata.
11+
* @author yingjianw
12+
* @since 1.2.0
13+
*/
14+
public interface MetadataPreMergeInterceptor {
15+
/**
16+
* Validate ObjectNode before storing it.
17+
* @param userMetadataService user metadata service
18+
* @param name qualified name
19+
* @param existingMetadata existing metadata
20+
* @param newMetadata newMetadata
21+
* @throws InvalidMetadataException business validation exception
22+
*/
23+
default void onWrite(final UserMetadataService userMetadataService,
24+
final QualifiedName name,
25+
final Optional<ObjectNode> existingMetadata,
26+
final Optional<ObjectNode> newMetadata) throws InvalidMetadataException {
27+
28+
}
29+
}

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/MetadataPreMergeInterceptorImpl.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.netflix.metacat.common.server.usermetadata;
2+
3+
/**
4+
* MetadataPreMergeInterceptorImpl.
5+
*
6+
* @author yingjianw
7+
* @since 1.2.0
8+
*/
9+
public class MetadataPreMergeInterceptorImpl implements MetadataPreMergeInterceptor {
10+
}

‎metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -684,11 +684,6 @@ public TableDto updateAndReturn(final QualifiedName name, final TableDto tableDt
684684
log.info("Saving user metadata for table {}", name);
685685
final long start = registry.clock().wallTime();
686686
userMetadataService.saveMetadata(metacatRequestContext.getUserName(), tableDto, true);
687-
final long vtts = MetacatUtils.getVtts(tableDto.getDefinitionMetadata());
688-
if (vtts > 0) {
689-
log.info("Received vtts update for {} to {}", name, vtts);
690-
userMetadataService.saveMetadata(metacatRequestContext.getUserName(), tableDto, true);
691-
}
692687

693688
final long duration = registry.clock().wallTime() - start;
694689
log.info("Time taken to save user metadata for table {} is {} ms", name, duration);

‎metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import com.netflix.metacat.common.server.converter.ConverterUtil;
1818
import com.netflix.metacat.common.server.properties.Config;
1919
import com.netflix.metacat.common.server.properties.MetacatProperties;
20+
import com.netflix.metacat.common.server.usermetadata.MetadataPreMergeInterceptor;
21+
import com.netflix.metacat.common.server.usermetadata.MetadataPreMergeInterceptorImpl;
2022
import com.netflix.metacat.common.server.usermetadata.UserMetadataService;
2123
import com.netflix.metacat.common.server.usermetadata.LookupService;
2224
import com.netflix.metacat.common.server.usermetadata.TagService;
@@ -54,23 +56,42 @@ public MetadataInterceptor businessMetadataManager(
5456
return new MetadataInterceptorImpl();
5557
}
5658

59+
/**
60+
* MetadataSQLInterceptor layer.
61+
* @return MetadataSQLInterceptor
62+
*/
63+
@Bean
64+
@ConditionalOnMissingBean(MetadataPreMergeInterceptor.class)
65+
public MetadataPreMergeInterceptor metadataPreMergeInterceptor(
66+
) {
67+
return new MetadataPreMergeInterceptorImpl();
68+
}
69+
5770
/**
5871
* User Metadata service.
5972
*
6073
* @param jdbcTemplate JDBC template
6174
* @param config System config to use
6275
* @param metacatJson Json Utilities to use
6376
* @param metadataInterceptor business metadata manager
77+
* @param metadataPreMergeInterceptor metadataPreMergeInterceptor
6478
* @return User metadata service based on MySql
6579
*/
6680
@Bean
6781
public UserMetadataService userMetadataService(
6882
@Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate,
6983
final Config config,
7084
final MetacatJson metacatJson,
71-
final MetadataInterceptor metadataInterceptor
72-
) {
73-
return new MysqlUserMetadataService(jdbcTemplate, metacatJson, config, metadataInterceptor);
85+
final MetadataInterceptor metadataInterceptor,
86+
final MetadataPreMergeInterceptor metadataPreMergeInterceptor
87+
) {
88+
return new MysqlUserMetadataService(
89+
jdbcTemplate,
90+
metacatJson,
91+
config,
92+
metadataInterceptor,
93+
metadataPreMergeInterceptor
94+
);
7495
}
7596

7697

‎metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MysqlUserMetadataService.java‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService;
3333
import com.netflix.metacat.common.server.usermetadata.GetMetadataInterceptorParameters;
3434
import com.netflix.metacat.common.server.usermetadata.MetadataInterceptor;
35+
import com.netflix.metacat.common.server.usermetadata.MetadataPreMergeInterceptor;
3536
import com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException;
3637
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
3738
import lombok.Data;
@@ -68,15 +69,16 @@
6869
@SuppressFBWarnings
6970
@Transactional("metadataTxManager")
7071
public class MysqlUserMetadataService extends BaseUserMetadataService {
71-
private static final String NAME_OWNER = "owner";
72-
private static final String NAME_USERID = "userId";
73-
private static final List<String> DEFINITION_METADATA_SORT_BY_COLUMNS = Arrays.asList(
72+
protected static final String NAME_OWNER = "owner";
73+
protected static final String NAME_USERID = "userId";
74+
protected static final List<String> DEFINITION_METADATA_SORT_BY_COLUMNS = Arrays.asList(
7475
"id", "date_created", "created_by", "last_updated_by", "name", "last_updated");
75-
private static final List<String> VALID_SORT_ORDER = Arrays.asList("ASC", "DESC");
76-
private final MetacatJson metacatJson;
77-
private final Config config;
78-
private JdbcTemplate jdbcTemplate;
79-
private final MetadataInterceptor metadataInterceptor;
76+
protected static final List<String> VALID_SORT_ORDER = Arrays.asList("ASC", "DESC");
77+
protected final MetacatJson metacatJson;
78+
protected final Config config;
79+
protected JdbcTemplate jdbcTemplate;
80+
protected final MetadataInterceptor metadataInterceptor;
81+
protected final MetadataPreMergeInterceptor metadataPreMergeInterceptor;
8082

8183
/**
8284
* Constructor.
@@ -85,18 +87,20 @@ public class MysqlUserMetadataService extends BaseUserMetadataService {
8587
* @param metacatJson json utility
8688
* @param config config
8789
* @param metadataInterceptor metadata interceptor
90+
* @param metadataPreMergeInterceptor metadataPreMergeInterceptor
8891
*/
8992
public MysqlUserMetadataService(
9093
final JdbcTemplate jdbcTemplate,
9194
final MetacatJson metacatJson,
9295
final Config config,
93-
final MetadataInterceptor metadataInterceptor
94-
96+
final MetadataInterceptor metadataInterceptor,
97+
final MetadataPreMergeInterceptor metadataPreMergeInterceptor
9598
) {
9699
this.metacatJson = metacatJson;
97100
this.config = config;
98101
this.jdbcTemplate = jdbcTemplate;
99102
this.metadataInterceptor = metadataInterceptor;
103+
this.metadataPreMergeInterceptor = metadataPreMergeInterceptor;
100104
}
101105

102106
@Override
@@ -363,6 +367,16 @@ public Optional<ObjectNode> getDefinitionMetadata(
363367
return retData;
364368
}
365369

370+
@Nonnull
371+
protected Optional<ObjectNode> getDefinitionMetadataForUpdate(
372+
@Nonnull final QualifiedName name) {
373+
final Optional<ObjectNode> retData = getJsonForKey(
374+
name.isPartitionDefinition()
375+
? SQL.GET_PARTITION_DEFINITION_METADATA : SQL.GET_DEFINITION_METADATA_FOR_UPDATE,
376+
name.toString());
377+
return retData;
378+
}
379+
366380
@Override
367381
@Transactional(readOnly = true)
368382
public List<QualifiedName> getDescendantDefinitionNames(@Nonnull final QualifiedName name) {
@@ -528,7 +542,7 @@ private Optional<ObjectNode> getJsonForKey(final String query, final String keyV
528542
* @param keyValues parameters
529543
* @return number of updated rows
530544
*/
531-
private int executeUpdateForKey(final String query, final String... keyValues) {
545+
protected int executeUpdateForKey(final String query, final String... keyValues) {
532546
try {
533547
final SqlParameterValue[] values =
534548
Arrays.stream(keyValues).map(keyValue -> new SqlParameterValue(Types.VARCHAR, keyValue))
@@ -541,7 +555,7 @@ private int executeUpdateForKey(final String query, final String... keyValues) {
541555
}
542556
}
543557

544-
private void throwIfPartitionDefinitionMetadataDisabled() {
558+
protected void throwIfPartitionDefinitionMetadataDisabled() {
545559
if (config.disablePartitionDefinitionMetadata()) {
546560
throw new MetacatBadRequestException("Partition Definition metadata updates are disabled");
547561
}
@@ -577,7 +591,15 @@ public void saveDefinitionMetadata(
577591
@Nonnull final String userId,
578592
@Nonnull final Optional<ObjectNode> metadata, final boolean merge)
579593
throws InvalidMetadataException {
580-
final Optional<ObjectNode> existingData = getDefinitionMetadata(name);
594+
final Optional<ObjectNode> existingData =
595+
config.isDefinitionMetadataSelectForUpdateEnabled()
596+
? getDefinitionMetadataForUpdate(name) : getDefinitionMetadata(name);
597+
metadataPreMergeInterceptor.onWrite(
598+
this,
599+
name,
600+
existingData,
601+
metadata
602+
);
581603
final int count;
582604
if (existingData.isPresent() && metadata.isPresent()) {
583605
ObjectNode merged = existingData.get();
@@ -983,6 +1005,8 @@ protected static class SQL {
9831005
"select uri name,data from data_metadata where uri in (%s)";
9841006
static final String GET_DEFINITION_METADATA =
9851007
"select name, data from definition_metadata where name=?";
1008+
static final String GET_DEFINITION_METADATA_FOR_UPDATE =
1009+
"select name, data from definition_metadata where name=? for update";
9861010
static final String GET_PARTITION_DEFINITION_METADATA =
9871011
"select name, data from partition_definition_metadata where name=?";
9881012
static final String GET_DEFINITION_METADATAS =

0 commit comments

Comments
 (0)