Skip to content

Commit 5d0a3a0

Browse files
yingjianwu98Yingjian Wu
yingjianwu98
and
Yingjian Wu
authored
Yingjianw/list db with follower reader timestamp (#589)
* follower_read_timestamp for list db calls * address comments --------- Co-authored-by: Yingjian Wu <[email protected]>
1 parent 9a9f271 commit 5d0a3a0

File tree

16 files changed

+436
-116
lines changed

16 files changed

+436
-116
lines changed

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -555,17 +555,31 @@ public interface Config {
555555
/**
556556
* Get the page size when listing table entities.
557557
*
558-
* @return True if it is.
558+
* @return size of the page
559559
*/
560560
int getListTableEntitiesPageSize();
561561

562562
/**
563563
* Get the page size when listing table names.
564564
*
565-
* @return True if it is.
565+
* @return size of the page
566566
*/
567567
int getListTableNamesPageSize();
568568

569+
/**
570+
* Get the page size when listing db entities.
571+
*
572+
* @return size of the page
573+
*/
574+
int getListDatabaseEntitiesPageSize();
575+
576+
/**
577+
* Get the page size when listing db names.
578+
*
579+
* @return size of the page
580+
*/
581+
int getListDatabaseNamesPageSize();
582+
569583
/**
570584
* Metadata query timeout in seconds.
571585
*

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,6 +649,16 @@ public int getListTableNamesPageSize() {
649649
return this.metacatProperties.getService().getListTableNamesPageSize();
650650
}
651651

652+
@Override
653+
public int getListDatabaseEntitiesPageSize() {
654+
return this.metacatProperties.getService().getListDatabaseEntitiesPageSize();
655+
}
656+
657+
@Override
658+
public int getListDatabaseNamesPageSize() {
659+
return this.metacatProperties.getService().getListDatabaseNamesPageSize();
660+
}
661+
652662
@Override
653663
public int getMetadataQueryTimeout() {
654664
return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds();

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ public class ServiceProperties {
3939
private boolean listDatabaseNameByDefaultOnGetCatalog = true;
4040
private int listTableEntitiesPageSize = 1000;
4141
private int listTableNamesPageSize = 10000;
42+
private int listDatabaseEntitiesPageSize = 1000;
43+
private int listDatabaseNamesPageSize = 10000;
4244

4345
/**
4446
* Max related properties.

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceFunctionalTest.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
2+
package com.netflix.metacat.connector.polaris;
3+
4+
import com.netflix.metacat.common.QualifiedName;
5+
import com.netflix.metacat.common.dto.Pageable;
6+
import com.netflix.metacat.common.dto.Sort;
7+
import com.netflix.metacat.common.dto.SortOrder;
8+
import com.netflix.metacat.common.server.connectors.model.DatabaseInfo;
9+
import com.netflix.metacat.connector.polaris.configs.PolarisPersistenceConfig;
10+
import lombok.extern.slf4j.Slf4j;
11+
import org.junit.Assert;
12+
import org.junit.jupiter.api.Test;
13+
import org.junit.jupiter.api.extension.ExtendWith;
14+
import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureDataJpa;
15+
import org.springframework.boot.test.context.SpringBootTest;
16+
import org.springframework.test.context.ActiveProfiles;
17+
import org.springframework.test.context.junit.jupiter.SpringExtension;
18+
19+
import java.util.Arrays;
20+
import java.util.List;
21+
22+
23+
/**
24+
* Test PolarisConnectorTableService.
25+
*/
26+
@Slf4j
27+
@ExtendWith(SpringExtension.class)
28+
@SpringBootTest(classes = {PolarisPersistenceConfig.class})
29+
@ActiveProfiles(profiles = {"polaris_functional_test"})
30+
@AutoConfigureDataJpa
31+
public class PolarisConnectorDatabaseServiceFunctionalTest extends PolarisConnectorDatabaseServiceTest {
32+
/**
33+
* Test SimpleDBList.
34+
*/
35+
@Test
36+
public void testSimpleListDb() {
37+
// Simulate a delay so that the dbs schema is visible
38+
TestUtil.simulateDelay();
39+
final DatabaseInfo db1 = DatabaseInfo.builder().name(DB1_QUALIFIED_NAME).uri("uri1").build();
40+
final DatabaseInfo db2 = DatabaseInfo.builder().name(DB2_QUALIFIED_NAME).uri("uri2").build();
41+
getPolarisDBService().create(getRequestContext(), db1);
42+
getPolarisDBService().create(getRequestContext(), db2);
43+
Assert.assertTrue(getPolarisDBService().exists(getRequestContext(), DB1_QUALIFIED_NAME));
44+
Assert.assertTrue(getPolarisDBService().exists(getRequestContext(), DB2_QUALIFIED_NAME));
45+
46+
// Since now list dbs use follower_read_timestamp, we will not immediately get the newly created dbs
47+
List<QualifiedName> dbNames =
48+
getPolarisDBService().listNames(
49+
getRequestContext(), QualifiedName.ofCatalog(CATALOG_NAME), null, null, null);
50+
List<DatabaseInfo> dbs =
51+
getPolarisDBService().list(
52+
getRequestContext(), QualifiedName.ofCatalog(CATALOG_NAME), null, null, null);
53+
Assert.assertTrue("Expected dbNames to be empty", dbNames.isEmpty());
54+
Assert.assertTrue("Expected dbs to be empty", dbs.isEmpty());
55+
56+
57+
// After sufficient time, the dbs should return using follower_read_timestamp
58+
TestUtil.simulateDelay();
59+
dbNames = getPolarisDBService().listNames(
60+
getRequestContext(), QualifiedName.ofCatalog(CATALOG_NAME), null, null, null);
61+
Assert.assertEquals(dbNames, Arrays.asList(DB1_QUALIFIED_NAME, DB2_QUALIFIED_NAME));
62+
dbs = getPolarisDBService().list(
63+
getRequestContext(), QualifiedName.ofCatalog(CATALOG_NAME), null, null, null);
64+
Assert.assertEquals(dbs, Arrays.asList(db1, db2));
65+
66+
// Test Prefix
67+
dbNames = getPolarisDBService().listNames(
68+
getRequestContext(),
69+
QualifiedName.ofCatalog(CATALOG_NAME), QualifiedName.ofDatabase(CATALOG_NAME, "db"),
70+
null,
71+
null);
72+
Assert.assertEquals(dbNames, Arrays.asList(DB1_QUALIFIED_NAME, DB2_QUALIFIED_NAME));
73+
dbs = getPolarisDBService().list(
74+
getRequestContext(),
75+
QualifiedName.ofCatalog(CATALOG_NAME),
76+
QualifiedName.ofDatabase(CATALOG_NAME, "db"),
77+
null,
78+
null);
79+
Assert.assertEquals(dbs, Arrays.asList(db1, db2));
80+
81+
dbNames = getPolarisDBService().listNames(
82+
getRequestContext(),
83+
QualifiedName.ofCatalog(CATALOG_NAME),
84+
QualifiedName.ofDatabase(CATALOG_NAME, "db1_"),
85+
null,
86+
null);
87+
Assert.assertEquals(dbNames, Arrays.asList(DB1_QUALIFIED_NAME));
88+
dbs = getPolarisDBService().list(
89+
getRequestContext(),
90+
QualifiedName.ofCatalog(CATALOG_NAME),
91+
QualifiedName.ofDatabase(CATALOG_NAME, "db1_"),
92+
null,
93+
null);
94+
Assert.assertEquals(dbs, Arrays.asList(db1));
95+
96+
// Test Order desc
97+
dbNames = getPolarisDBService().listNames(
98+
getRequestContext(),
99+
QualifiedName.ofCatalog(CATALOG_NAME),
100+
null,
101+
new Sort("name", SortOrder.DESC),
102+
null);
103+
Assert.assertEquals(dbNames, Arrays.asList(DB2_QUALIFIED_NAME, DB1_QUALIFIED_NAME));
104+
dbs = getPolarisDBService().list(
105+
getRequestContext(),
106+
QualifiedName.ofCatalog(CATALOG_NAME),
107+
null,
108+
new Sort("name", SortOrder.DESC),
109+
null);
110+
Assert.assertEquals(dbs, Arrays.asList(db2, db1));
111+
112+
// Test pageable
113+
dbNames = getPolarisDBService().listNames(
114+
getRequestContext(),
115+
QualifiedName.ofCatalog(CATALOG_NAME),
116+
null,
117+
null,
118+
new Pageable(5, 0));
119+
Assert.assertEquals(dbNames, Arrays.asList(DB1_QUALIFIED_NAME, DB2_QUALIFIED_NAME));
120+
dbs = getPolarisDBService().list(
121+
getRequestContext(),
122+
QualifiedName.ofCatalog(CATALOG_NAME),
123+
null,
124+
null,
125+
new Pageable(5, 0));
126+
Assert.assertEquals(dbs, Arrays.asList(db1, db2));
127+
128+
dbNames = getPolarisDBService().listNames(
129+
getRequestContext(),
130+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(1, 0));
131+
Assert.assertEquals(dbNames, Arrays.asList(DB1_QUALIFIED_NAME));
132+
dbs = getPolarisDBService().list(
133+
getRequestContext(),
134+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(1, 0));
135+
Assert.assertEquals(dbs, Arrays.asList(db1));
136+
137+
dbNames = getPolarisDBService().listNames(
138+
getRequestContext(),
139+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(1, 1));
140+
Assert.assertEquals(dbNames, Arrays.asList(DB2_QUALIFIED_NAME));
141+
dbs = getPolarisDBService().list(
142+
getRequestContext(),
143+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(1, 1));
144+
Assert.assertEquals(dbs, Arrays.asList(db2));
145+
146+
dbNames = getPolarisDBService().listNames(
147+
getRequestContext(),
148+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(5, 1));
149+
Assert.assertEquals(dbNames, Arrays.asList(DB2_QUALIFIED_NAME));
150+
dbs = getPolarisDBService().list(
151+
getRequestContext(),
152+
QualifiedName.ofCatalog(CATALOG_NAME), null, null, new Pageable(5, 1));
153+
Assert.assertEquals(dbs, Arrays.asList(db2));
154+
}
155+
}
156+

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,7 @@ public void testGetTableNames() {
5757
.build();
5858
getPolarisTableService().create(getRequestContext(), tableInfo3);
5959

60-
try {
61-
// pause execution for 10000 milliseconds (10 seconds)
62-
Thread.sleep(10000);
63-
} catch (InterruptedException e) {
64-
log.debug("Sleep was interrupted");
65-
}
60+
TestUtil.simulateDelay();
6661

6762
final List<QualifiedName> tables = getPolarisTableService()
6863
.getTableNames(getRequestContext(), DB_QUALIFIED_NAME, "", -1);
@@ -77,12 +72,7 @@ public void testGetTableNames() {
7772
public void testListTablesEmpty() {
7873
final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "");
7974

80-
try {
81-
// pause execution for 10000 milliseconds (10 seconds)
82-
Thread.sleep(10000);
83-
} catch (InterruptedException e) {
84-
log.debug("Sleep was interrupted");
85-
}
75+
TestUtil.simulateDelay();
8676

8777
final List<QualifiedName> names = getPolarisTableService().listNames(
8878
getRequestContext(), DB_QUALIFIED_NAME, qualifiedName,
@@ -102,12 +92,7 @@ public void testTableCreationAndList() {
10292
.build();
10393
getPolarisTableService().create(getRequestContext(), tableInfo);
10494

105-
try {
106-
// pause execution for 10000 milliseconds (10 seconds)
107-
Thread.sleep(10000);
108-
} catch (InterruptedException e) {
109-
log.debug("Sleep was interrupted");
110-
}
95+
TestUtil.simulateDelay();
11196

11297
final List<QualifiedName> names = getPolarisTableService().listNames(
11398
getRequestContext(), DB_QUALIFIED_NAME, qualifiedName,
@@ -136,12 +121,7 @@ public void testList() {
136121

137122
final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "");
138123

139-
try {
140-
// pause execution for 10000 milliseconds (10 seconds)
141-
Thread.sleep(10000);
142-
} catch (InterruptedException e) {
143-
log.debug("Sleep was interrupted");
144-
}
124+
TestUtil.simulateDelay();
145125

146126
List<TableInfo> tables = this.getPolarisTableService().list(
147127
this.getRequestContext(), DB_QUALIFIED_NAME, qualifiedName, new Sort(null, SortOrder.ASC),

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisStoreConnectorFunctionalTest.java‎

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.netflix.metacat.connector.polaris.configs.PolarisPersistenceConfig;
55
import com.netflix.metacat.connector.polaris.store.PolarisStoreConnectorTest;
66
import com.netflix.metacat.connector.polaris.store.entities.PolarisTableEntity;
7+
import com.netflix.metacat.connector.polaris.store.entities.PolarisDatabaseEntity;
78
import lombok.extern.slf4j.Slf4j;
89
import org.junit.Assert;
910
import org.junit.jupiter.api.Test;
@@ -13,7 +14,9 @@
1314
import org.springframework.test.context.ActiveProfiles;
1415
import org.springframework.test.context.junit.jupiter.SpringExtension;
1516

17+
import java.util.Arrays;
1618
import java.util.List;
19+
import java.util.stream.Collectors;
1720

1821
/**
1922
* Test persistence operations on Database objects.
@@ -42,12 +45,7 @@ public void testPaginatedFetch() {
4245
createTable(dbName, tblNameB);
4346
createTable(dbName, tblNameC);
4447

45-
try {
46-
// pause execution for 10000 milliseconds (10 seconds)
47-
Thread.sleep(10000);
48-
} catch (InterruptedException e) {
49-
log.debug("Sleep was interrupted");
50-
}
48+
TestUtil.simulateDelay();
5149

5250
tblNames = getPolarisConnector().getTables(dbName, "", 1000);
5351
Assert.assertEquals(3, tblNames.size());
@@ -65,12 +63,7 @@ public void testGetTableEntities() {
6563
final String dbName = generateDatabaseName();
6664
createDB(dbName);
6765

68-
try {
69-
// pause execution for 10000 milliseconds (10 seconds)
70-
Thread.sleep(10000);
71-
} catch (InterruptedException e) {
72-
log.debug("Sleep was interrupted");
73-
}
66+
TestUtil.simulateDelay();
7467

7568
// Test when db is empty
7669
List<PolarisTableEntity> entities = getPolarisConnector().getTableEntities(dbName, "", 1);
@@ -85,12 +78,7 @@ public void testGetTableEntities() {
8578
createTable(dbName, tblNameB);
8679
createTable(dbName, tblNameC);
8780

88-
try {
89-
// pause execution for 10000 milliseconds (10 seconds)
90-
Thread.sleep(10000);
91-
} catch (InterruptedException e) {
92-
log.debug("Sleep was interrupted");
93-
}
81+
TestUtil.simulateDelay();
9482

9583
// Test pagination and sort
9684
entities = getPolarisConnector().getTableEntities(dbName, "", 1);
@@ -117,4 +105,40 @@ public void testGetTableEntities() {
117105
Assert.assertEquals(tblNameB, entities.get(1).getTblName());
118106
Assert.assertEquals(tblNameC, entities.get(2).getTblName());
119107
}
108+
109+
/**
110+
* test list database with different db page size config.
111+
*/
112+
@Test
113+
public void testListDbPage() {
114+
createDB("db1");
115+
createDB("db2");
116+
createDB("db3");
117+
118+
TestUtil.simulateDelay();
119+
120+
List<String> dbNames = getPolarisConnector().getDatabaseNames("db", null, 1);
121+
List<PolarisDatabaseEntity> dbs = getPolarisConnector().getDatabases("db", null, 1);
122+
Assert.assertEquals("Expected dbNames ", Arrays.asList("db1", "db2", "db3"), dbNames);
123+
Assert.assertEquals("Expected dbs ", Arrays.asList("db1", "db2", "db3"),
124+
dbs.stream().map(PolarisDatabaseEntity::getDbName).collect(Collectors.toList()));
125+
126+
dbNames = getPolarisConnector().getDatabaseNames("db", null, 2);
127+
dbs = getPolarisConnector().getDatabases("db", null, 2);
128+
Assert.assertEquals("Expected dbNames ", Arrays.asList("db1", "db2", "db3"), dbNames);
129+
Assert.assertEquals("Expected dbs ", Arrays.asList("db1", "db2", "db3"),
130+
dbs.stream().map(PolarisDatabaseEntity::getDbName).collect(Collectors.toList()));
131+
132+
dbNames = getPolarisConnector().getDatabaseNames("db", null, 3);
133+
dbs = getPolarisConnector().getDatabases("db", null, 3);
134+
Assert.assertEquals("Expected dbNames ", Arrays.asList("db1", "db2", "db3"), dbNames);
135+
Assert.assertEquals("Expected dbs ", Arrays.asList("db1", "db2", "db3"),
136+
dbs.stream().map(PolarisDatabaseEntity::getDbName).collect(Collectors.toList()));
137+
138+
dbNames = getPolarisConnector().getDatabaseNames("db", null, 4);
139+
dbs = getPolarisConnector().getDatabases("db", null, 4);
140+
Assert.assertEquals("Expected dbNames ", Arrays.asList("db1", "db2", "db3"), dbNames);
141+
Assert.assertEquals("Expected dbs ", Arrays.asList("db1", "db2", "db3"),
142+
dbs.stream().map(PolarisDatabaseEntity::getDbName).collect(Collectors.toList()));
143+
}
120144
}

0 commit comments

Comments
 (0)