Skip to content

Commit e8ea9f9

Browse files
yingjianwu98Yingjian Wu
yingjianwu98
and
Yingjian Wu
authored
Fix setTag logic to prevent high cpu load on rds (#572)
* remove unused code + use insert if not exist for values_string instead of loading from memory + test * address comments --------- Co-authored-by: Yingjian Wu <[email protected]>
1 parent 6d7bfca commit e8ea9f9

File tree

5 files changed

+187
-89
lines changed

5 files changed

+187
-89
lines changed

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/LookupService.java‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,14 @@
3131
public interface LookupService {
3232
/**
3333
* Returns the lookup for the given <code>name</code>.
34+
* If includeValues = true, we will set the Lookup values with the associated name, otherwise empty set
3435
*
3536
* @param name lookup name
37+
* @param includeValues whether we should set the values or not in the Lookup Object
3638
* @return lookup
3739
*/
3840
@Nullable
39-
default Lookup get(final String name) {
41+
default Lookup get(final String name, final boolean includeValues) {
4042
return null;
4143
}
4244

@@ -75,22 +77,20 @@ default Set<String> getValues(final Long lookupId) {
7577
*
7678
* @param name lookup name
7779
* @param values multiple values
80+
* @param includeValues whether to populate the values field in the Lookup Object
7881
* @return updated lookup
7982
*/
80-
@Nullable
81-
default Lookup setValues(final String name, final Set<String> values) {
82-
return null;
83-
}
8483

8584
/**
8685
* Saves the lookup value.
8786
*
8887
* @param name lookup name
8988
* @param values multiple values
89+
* @param includeValues whether to include values in the final Lookup Object
9090
* @return updated lookup
9191
*/
9292
@Nullable
93-
default Lookup addValues(final String name, final Set<String> values) {
93+
default Lookup addValues(final String name, final Set<String> values, boolean includeValues) {
9494
return null;
9595
}
9696

@@ -99,11 +99,7 @@ default Lookup addValues(final String name, final Set<String> values) {
9999
*
100100
* @param name lookup name
101101
* @param value lookup value
102+
* @param includeValues whether to return lookup value in the Lookup Object
102103
* @return updated lookup
103104
*/
104-
@Nullable
105-
default Lookup setValue(final String name, final String value) {
106-
return null;
107-
}
108-
109105
}

‎metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1771,4 +1771,83 @@ class MetacatSmokeSpec extends Specification {
17711771
'embedded-hive-metastore/invalid/dm/vm',
17721772
'embedded-hive-metastore/invalid/dm/vm=1']
17731773
}
1774+
1775+
def "Test Set tags for all scenarios"() {
1776+
setup:
1777+
// The setup already contains the following 7 tags
1778+
// [data-category:non-customer, test, test_tag, unused, do_not_drop, iceberg_migration_do_not_modify, do_not_rename]
1779+
def catalogName = "hive-metastore"
1780+
def databaseName = "default"
1781+
def catalog = QualifiedName.ofCatalog(catalogName)
1782+
def database = QualifiedName.ofDatabase(catalogName, databaseName)
1783+
1784+
def table1Name = "table1"
1785+
def table1Tags = ['table1', 'mock'] as Set
1786+
1787+
def table2Name = "table2"
1788+
def table2Tags = ['table2', 'mock'] as Set
1789+
1790+
def table3Name = "table3"
1791+
def table3Tags = ['table3', 'bdow'] as Set
1792+
when:
1793+
// add 2 new tags on table1
1794+
createTable(catalogName, databaseName, table1Name)
1795+
tagApi.setTableTags(catalogName, databaseName, table1Name, table1Tags)
1796+
then:
1797+
tagApi.getTags().size() == 9
1798+
tagApi.list(['table1', 'mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 1
1799+
tagApi.search('mock', null, null, null).size() == 1
1800+
tagApi.search('table1', null, null, null).size() == 1
1801+
metacatJson.convertValue(api.getTable(catalogName, databaseName, table1Name, false, true, true).getDefinitionMetadata().get('tags'), Set.class) == table1Tags
1802+
when:
1803+
// add an existing tag mock but a new tag on table 2
1804+
createTable(catalogName, databaseName, table2Name)
1805+
tagApi.setTableTags(catalogName, databaseName, table2Name, table2Tags)
1806+
then:
1807+
tagApi.getTags().size() == 10
1808+
tagApi.list(['table2'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 1
1809+
tagApi.list(['mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1810+
tagApi.list(['table1', 'mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1811+
tagApi.search('mock', null, null, null).size() == 2
1812+
tagApi.search('table1', null, null, null).size() == 1
1813+
tagApi.search('table2', null, null, null).size() == 1
1814+
metacatJson.convertValue(api.getTable(catalogName, databaseName, table2Name, false, true, true).getDefinitionMetadata().get('tags'), Set.class) == table2Tags
1815+
when:
1816+
// delete table1, for now even if tag table1 is only applied to table1, we will not remove the tag from our db
1817+
api.deleteTable(catalogName, databaseName, "table1")
1818+
then:
1819+
tagApi.getTags().size() == 10
1820+
tagApi.list(['table1'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 1
1821+
tagApi.list(['table2', 'mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1822+
tagApi.list(['mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1823+
tagApi.search('mock', null, null, null).size() == 2
1824+
tagApi.search('table1', null, null, null).size() == 1
1825+
tagApi.search('table2', null, null, null).size() == 1
1826+
when:
1827+
// add tags on catalog database and table level through setTags api
1828+
// create 1 new catalogTag on catalog
1829+
// create 1 new databaseTag on database
1830+
// create 2 new tableTags on table3
1831+
tagApi.setTags(TagCreateRequestDto.builder().name(catalog).tags(['mock', 'catalogTag'] as List<String>).build())
1832+
tagApi.setTags(TagCreateRequestDto.builder().name(database).tags(['mock', 'databaseTag'] as List<String>).build())
1833+
createTable(catalogName, databaseName, "table3")
1834+
def table3 = QualifiedName.ofTable(catalogName, databaseName, table3Name)
1835+
tagApi.setTags(TagCreateRequestDto.builder().name(table3).tags(table3Tags as List<String>).build())
1836+
then:
1837+
tagApi.getTags().size() == 14
1838+
tagApi.list(['table2', 'mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1839+
tagApi.list(['table3', 'bdow'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 1
1840+
tagApi.list(['mock'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE).size() == 2
1841+
tagApi.list(['mock'] as Set<String>, null, null, null, null, QualifiedName.Type.DATABASE).size() == 1
1842+
tagApi.list(['mock'] as Set<String>, null, null, null, null, QualifiedName.Type.CATALOG).size() == 1
1843+
tagApi.list(['mock'] as Set<String>, null, null, null, null, null).size() == 4
1844+
tagApi.search('mock', null, null, null).size() == 4
1845+
tagApi.search('table3', null, null, null).size() == 1
1846+
tagApi.search('bdow', null, null, null).size() == 1
1847+
tagApi.search('table2', null, null, null).size() == 1
1848+
metacatJson.convertValue(api.getTable(catalogName, databaseName, table3Name, false, true, true).getDefinitionMetadata().get('tags'), Set.class) == table3Tags
1849+
cleanup:
1850+
api.deleteTable(catalogName, databaseName, "table2")
1851+
api.deleteTable(catalogName, databaseName, "table3")
1852+
}
17741853
}

‎metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlLookupService.java‎

Lines changed: 18 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313

1414
package com.netflix.metacat.metadata.mysql;
1515

16-
import com.google.common.base.Joiner;
1716
import com.google.common.collect.Sets;
1817
import com.netflix.metacat.common.server.model.Lookup;
1918
import com.netflix.metacat.common.server.properties.Config;
@@ -23,7 +22,6 @@
2322
import lombok.extern.slf4j.Slf4j;
2423
import org.springframework.dao.EmptyResultDataAccessException;
2524
import org.springframework.jdbc.core.JdbcTemplate;
26-
import org.springframework.jdbc.core.SqlParameterValue;
2725
import org.springframework.jdbc.support.GeneratedKeyHolder;
2826
import org.springframework.jdbc.support.KeyHolder;
2927
import org.springframework.transaction.annotation.Transactional;
@@ -32,6 +30,7 @@
3230
import java.sql.SQLException;
3331
import java.sql.Statement;
3432
import java.sql.Types;
33+
import java.util.Collections;
3534
import java.util.Set;
3635
import java.util.stream.Collectors;
3736

@@ -50,8 +49,9 @@ public class MySqlLookupService implements LookupService {
5049
+ " values (?,0,?,?,?,now(),now())";
5150
private static final String SQL_INSERT_LOOKUP_VALUES =
5251
"insert into lookup_values( lookup_id, values_string) values (?,?)";
53-
private static final String SQL_DELETE_LOOKUP_VALUES =
54-
"delete from lookup_values where lookup_id=? and values_string in (%s)";
52+
private static final String SQL_INSERT_LOOKUP_VALUE_IF_NOT_EXIST =
53+
"INSERT IGNORE INTO lookup_values (lookup_id, values_string) VALUES (?, ?)";
54+
5555
private static final String SQL_GET_LOOKUP_VALUES =
5656
"select values_string value from lookup_values where lookup_id=?";
5757
private static final String SQL_GET_LOOKUP_VALUES_BY_NAME =
@@ -79,7 +79,7 @@ public MySqlLookupService(final Config config, final JdbcTemplate jdbcTemplate)
7979
*/
8080
@Override
8181
@Transactional(readOnly = true)
82-
public Lookup get(final String name) {
82+
public Lookup get(final String name, final boolean includeValues) {
8383
try {
8484
return jdbcTemplate.queryForObject(
8585
SQL_GET_LOOKUP,
@@ -93,7 +93,7 @@ public Lookup get(final String name) {
9393
lookup.setLastUpdated(rs.getDate("lastUpdated"));
9494
lookup.setLastUpdatedBy(rs.getString("lastUpdatedBy"));
9595
lookup.setDateCreated(rs.getDate("dateCreated"));
96-
lookup.setValues(getValues(rs.getLong("id")));
96+
lookup.setValues(includeValues ? getValues(rs.getLong("id")) : Collections.EMPTY_SET);
9797
return lookup;
9898
});
9999
} catch (EmptyResultDataAccessException e) {
@@ -162,61 +162,22 @@ public Set<String> getValues(final String name) {
162162
}
163163
}
164164

165-
/**
166-
* Saves the lookup value.
167-
*
168-
* @param name lookup name
169-
* @param values multiple values
170-
* @return returns the lookup with the given name.
171-
*/
172-
@Override
173-
public Lookup setValues(final String name, final Set<String> values) {
174-
try {
175-
final Lookup lookup = findOrCreateLookupByName(name);
176-
final Set<String> inserts;
177-
Set<String> deletes = Sets.newHashSet();
178-
final Set<String> lookupValues = lookup.getValues();
179-
if (lookupValues == null || lookupValues.isEmpty()) {
180-
inserts = values;
181-
} else {
182-
inserts = Sets.difference(values, lookupValues).immutableCopy();
183-
deletes = Sets.difference(lookupValues, values).immutableCopy();
184-
}
185-
lookup.setValues(values);
186-
if (!inserts.isEmpty()) {
187-
insertLookupValues(lookup.getId(), inserts);
188-
}
189-
if (!deletes.isEmpty()) {
190-
deleteLookupValues(lookup.getId(), deletes);
191-
}
192-
return lookup;
193-
} catch (Exception e) {
194-
final String message = String.format("Failed to set the lookup values for name %s", name);
195-
log.error(message, e);
196-
throw new UserMetadataServiceException(message, e);
197-
}
198-
}
199-
200-
private void insertLookupValues(final Long id, final Set<String> inserts) {
201-
jdbcTemplate.batchUpdate(SQL_INSERT_LOOKUP_VALUES, inserts.stream().map(insert -> new Object[]{id, insert})
165+
private void insertLookupValuesIfNotExist(final Long id, final Set<String> inserts) {
166+
jdbcTemplate.batchUpdate(SQL_INSERT_LOOKUP_VALUE_IF_NOT_EXIST,
167+
inserts.stream().map(insert -> new Object[]{id, insert})
202168
.collect(Collectors.toList()), new int[]{Types.BIGINT, Types.VARCHAR});
203169
}
204170

205-
private void deleteLookupValues(final Long id, final Set<String> deletes) {
206-
jdbcTemplate.update(
207-
String.format(SQL_DELETE_LOOKUP_VALUES, "'" + Joiner.on("','").skipNulls().join(deletes) + "'"),
208-
new SqlParameterValue(Types.BIGINT, id));
209-
}
210-
211171
/**
212172
* findOrCreateLookupByName.
213173
*
214174
* @param name name to find or create
175+
* @param includeValues whether to include the values in the Lookup Object
215176
* @return Look up object
216177
* @throws SQLException sql exception
217178
*/
218-
private Lookup findOrCreateLookupByName(final String name) throws SQLException {
219-
Lookup lookup = get(name);
179+
private Lookup findOrCreateLookupByName(final String name, final boolean includeValues) throws SQLException {
180+
Lookup lookup = get(name, includeValues);
220181
if (lookup == null) {
221182
final KeyHolder holder = new GeneratedKeyHolder();
222183
jdbcTemplate.update(connection -> {
@@ -244,20 +205,14 @@ private Lookup findOrCreateLookupByName(final String name) throws SQLException {
244205
* @return returns the lookup with the given name.
245206
*/
246207
@Override
247-
public Lookup addValues(final String name, final Set<String> values) {
208+
public Lookup addValues(final String name, final Set<String> values, final boolean includeValues) {
248209
try {
249-
final Lookup lookup = findOrCreateLookupByName(name);
250-
251-
final Set<String> inserts;
252-
final Set<String> lookupValues = lookup.getValues();
253-
if (lookupValues == null || lookupValues.isEmpty()) {
254-
inserts = values;
255-
lookup.setValues(values);
256-
} else {
257-
inserts = Sets.difference(values, lookupValues);
210+
final Lookup lookup = findOrCreateLookupByName(name, includeValues);
211+
if (!values.isEmpty()) {
212+
insertLookupValuesIfNotExist(lookup.getId(), values);
258213
}
259-
if (!inserts.isEmpty()) {
260-
insertLookupValues(lookup.getId(), inserts);
214+
if (includeValues) {
215+
lookup.getValues().addAll(values);
261216
}
262217
return lookup;
263218
} catch (Exception e) {
@@ -266,16 +221,4 @@ public Lookup addValues(final String name, final Set<String> values) {
266221
throw new UserMetadataServiceException(message, e);
267222
}
268223
}
269-
270-
/**
271-
* Saves the lookup value.
272-
*
273-
* @param name lookup name
274-
* @param value lookup value
275-
* @return returns the lookup with the given name.
276-
*/
277-
@Override
278-
public Lookup setValue(final String name, final String value) {
279-
return setValues(name, Sets.newHashSet(value));
280-
}
281224
}

‎metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,9 @@ public MySqlTagService(
124124
this.userMetadataService = Preconditions.checkNotNull(userMetadataService, "userMetadataService is required");
125125
}
126126

127-
private Lookup addTags(final Set<String> tags) {
127+
private Lookup addTags(final Set<String> tags, final boolean includeValues) {
128128
try {
129-
return lookupService.addValues(LOOKUP_NAME_TAG, tags);
129+
return lookupService.addValues(LOOKUP_NAME_TAG, tags, includeValues);
130130
} catch (Exception e) {
131131
final String message = String.format("Failed adding the tags %s", tags);
132132
log.error(message, e);
@@ -396,7 +396,7 @@ public List<QualifiedName> search(
396396
@Override
397397
public Set<String> setTags(final QualifiedName name, final Set<String> tags,
398398
final boolean updateUserMetadata) {
399-
addTags(tags);
399+
addTags(tags, false);
400400
try {
401401
final TagItem tagItem = findOrCreateTagItemByName(name.toString());
402402
final Set<String> inserts;

0 commit comments

Comments
 (0)