Skip to content

Commit c66f3b0

Browse files
author
Yingjian Wu
committed
wip
add test fix test fix test fix schema fix schema wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip
1 parent a021467 commit c66f3b0

29 files changed

+783
-451
lines changed

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceFunctionalTest.java‎

Lines changed: 166 additions & 68 deletions
Large diffs are not rendered by default.

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java‎

Lines changed: 175 additions & 38 deletions
Large diffs are not rendered by default.

‎metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnectorFunctionalTest.java‎

Lines changed: 96 additions & 61 deletions
Large diffs are not rendered by default.

‎metacat-connector-polaris/src/functionalTest/resources/aurora-schema.sql‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎metacat-connector-polaris/src/functionalTest/resources/schema.sql‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,32 @@
1-
drop table IF EXISTS TBLS;
2-
drop table IF EXISTS DBS;
1+
DROP TABLE IF EXISTS TBLS;
2+
DROP TABLE IF EXISTS DBS;
33

4-
create table DBS (
5-
version bigint not null,
6-
id uuid default gen_random_uuid() not null primary key,
7-
name varchar(255) not null unique,
8-
location varchar(8192),
9-
created_by STRING(255),
10-
created_date TIMESTAMP not null,
11-
last_updated_by STRING(255),
12-
last_updated_date TIMESTAMP not null
4+
CREATE TABLE DBS (
5+
version BIGINT NOT NULL,
6+
id UUID DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY,
7+
catalog_name VARCHAR(255),
8+
name VARCHAR(255) NOT NULL,
9+
location VARCHAR(8192),
10+
created_by VARCHAR(255),
11+
created_date TIMESTAMP NOT NULL,
12+
last_updated_by VARCHAR(255),
13+
last_updated_date TIMESTAMP NOT NULL,
14+
CONSTRAINT unique_catalog_name_db UNIQUE (catalog_name, name)
1315
);
1416

15-
create table TBLS (
16-
version bigint not null,
17-
id uuid default gen_random_uuid() not null primary key,
18-
db_name varchar(255) not null,
19-
tbl_name varchar(255) not null,
20-
previous_metadata_location varchar(8192),
21-
metadata_location varchar(8192),
22-
params TEXT,
23-
constraint uniq_name unique(db_name, tbl_name),
24-
created_by STRING(255),
25-
created_date TIMESTAMP not null,
26-
last_updated_by STRING(255),
27-
last_updated_date TIMESTAMP not null,
28-
foreign key (db_name) references DBS(name) ON DELETE CASCADE ON UPDATE CASCADE
17+
CREATE TABLE TBLS (
18+
version BIGINT NOT NULL,
19+
id UUID DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY,
20+
catalog_name VARCHAR(255),
21+
db_name VARCHAR(255) NOT NULL,
22+
tbl_name VARCHAR(255) NOT NULL,
23+
previous_metadata_location VARCHAR(8192),
24+
metadata_location VARCHAR(8192),
25+
params TEXT,
26+
created_by VARCHAR(255),
27+
created_date TIMESTAMP NOT NULL,
28+
last_updated_by VARCHAR(255),
29+
last_updated_date TIMESTAMP NOT NULL,
30+
CONSTRAINT unique_catalog_db_tbl UNIQUE (catalog_name, db_name, tbl_name),
31+
CONSTRAINT fk_tbls_db FOREIGN KEY (catalog_name, db_name) REFERENCES DBS(catalog_name, name) ON DELETE CASCADE ON UPDATE CASCADE
2932
);

‎metacat-connector-polaris/src/functionalTest/scripts/datastores/aurora/docker-entrypoint-initdb-primary.d/schema.sql‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,38 @@
11
-- Enable the uuid-ossp extension for UUID generation
22
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
33

4-
drop table IF EXISTS TBLS;
5-
drop table IF EXISTS DBS;
4+
DROP TABLE IF EXISTS TBLS;
5+
DROP TABLE IF EXISTS DBS;
66

77
create user replicator with replication encrypted password 'replicator_password';
88
select pg_create_physical_replication_slot('replication_slot');
99

10-
create table DBS (
11-
version bigint not null,
12-
id uuid default uuid_generate_v4() not null primary key,
13-
name varchar(255) not null unique,
10+
CREATE TABLE DBS (
11+
version bigint NOT NULL,
12+
id uuid DEFAULT uuid_generate_v4() NOT NULL PRIMARY KEY,
13+
catalog_name varchar(255),
14+
name varchar(255) NOT NULL,
1415
location varchar(8192),
1516
created_by varchar(255),
16-
created_date TIMESTAMP not null,
17+
created_date TIMESTAMP NOT NULL,
1718
last_updated_by varchar(255),
18-
last_updated_date TIMESTAMP not null
19+
last_updated_date TIMESTAMP NOT NULL,
20+
CONSTRAINT unique_catalog_name_db UNIQUE (catalog_name, name)
1921
);
2022

21-
create table TBLS (
22-
version bigint not null,
23-
id uuid default uuid_generate_v4() not null primary key,
24-
db_name varchar(255) not null,
25-
tbl_name varchar(255) not null,
23+
CREATE TABLE TBLS (
24+
version bigint NOT NULL,
25+
id uuid DEFAULT uuid_generate_v4() NOT NULL PRIMARY KEY,
26+
catalog_name varchar(255),
27+
db_name varchar(255) NOT NULL,
28+
tbl_name varchar(255) NOT NULL,
2629
previous_metadata_location varchar(8192),
2730
metadata_location varchar(8192),
2831
params text,
2932
created_by varchar(255),
30-
created_date TIMESTAMP not null,
33+
created_date TIMESTAMP NOT NULL,
3134
last_updated_by varchar(255),
32-
last_updated_date TIMESTAMP not null,
33-
constraint uniq_name unique(db_name, tbl_name),
34-
foreign key (db_name) references DBS(name) ON DELETE RESTRICT ON UPDATE CASCADE
35+
last_updated_date TIMESTAMP NOT NULL,
36+
CONSTRAINT unique_catalog_db_tbl UNIQUE (catalog_name, db_name, tbl_name),
37+
CONSTRAINT fk_tbls_db FOREIGN KEY (catalog_name, db_name) REFERENCES DBS(catalog_name, name) ON DELETE RESTRICT ON UPDATE CASCADE
3538
);

‎metacat-connector-polaris/src/functionalTest/scripts/datastores/aurora/docker-entrypoint-initdb-reader.d/schema.sql‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,38 @@
11
-- Enable the uuid-ossp extension for UUID generation
22
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
33

4-
drop table IF EXISTS TBLS;
5-
drop table IF EXISTS DBS;
4+
DROP TABLE IF EXISTS TBLS;
5+
DROP TABLE IF EXISTS DBS;
66

7-
create table DBS (
8-
version bigint not null,
9-
id uuid default uuid_generate_v4() not null primary key,
10-
name varchar(255) not null unique,
7+
create user replicator with replication encrypted password 'replicator_password';
8+
select pg_create_physical_replication_slot('replication_slot');
9+
10+
CREATE TABLE DBS (
11+
version bigint NOT NULL,
12+
id uuid DEFAULT uuid_generate_v4() NOT NULL PRIMARY KEY,
13+
catalog_name varchar(255),
14+
name varchar(255) NOT NULL,
1115
location varchar(8192),
1216
created_by varchar(255),
13-
created_date TIMESTAMP not null,
17+
created_date TIMESTAMP NOT NULL,
1418
last_updated_by varchar(255),
15-
last_updated_date TIMESTAMP not null
19+
last_updated_date TIMESTAMP NOT NULL,
20+
CONSTRAINT unique_catalog_name_db UNIQUE (catalog_name, name)
1621
);
1722

18-
create table TBLS (
19-
version bigint not null,
20-
id uuid default uuid_generate_v4() not null primary key,
21-
db_name varchar(255) not null,
22-
tbl_name varchar(255) not null,
23+
CREATE TABLE TBLS (
24+
version bigint NOT NULL,
25+
id uuid DEFAULT uuid_generate_v4() NOT NULL PRIMARY KEY,
26+
catalog_name varchar(255),
27+
db_name varchar(255) NOT NULL,
28+
tbl_name varchar(255) NOT NULL,
2329
previous_metadata_location varchar(8192),
2430
metadata_location varchar(8192),
2531
params text,
2632
created_by varchar(255),
27-
created_date TIMESTAMP not null,
33+
created_date TIMESTAMP NOT NULL,
2834
last_updated_by varchar(255),
29-
last_updated_date TIMESTAMP not null,
30-
constraint uniq_name unique(db_name, tbl_name),
31-
foreign key (db_name) references DBS(name) ON DELETE RESTRICT ON UPDATE CASCADE
35+
last_updated_date TIMESTAMP NOT NULL,
36+
CONSTRAINT unique_catalog_db_tbl UNIQUE (catalog_name, db_name, tbl_name),
37+
CONSTRAINT fk_tbls_db FOREIGN KEY (catalog_name, db_name) REFERENCES DBS(catalog_name, name) ON DELETE RESTRICT ON UPDATE CASCADE
3238
);

‎metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseService.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void create(final ConnectorRequestContext context, final DatabaseInfo dat
6666
try {
6767
final String location = databaseInfo.getUri() == null
6868
? this.defaultLocationPrefix + name.getDatabaseName() + DEFAULT_LOCATION_SUFFIX : databaseInfo.getUri();
69-
this.polarisStoreService.createDatabase(name.getDatabaseName(), location, createdBy);
69+
this.polarisStoreService.createDatabase(name.getCatalogName(), name.getDatabaseName(), location, createdBy);
7070
} catch (DataIntegrityViolationException exception) {
7171
throw new InvalidMetaException(name, exception);
7272
} catch (Exception exception) {
@@ -85,7 +85,7 @@ public void delete(final ConnectorRequestContext context, final QualifiedName na
8585
throw new DatabaseNotFoundException(name);
8686
}
8787
try {
88-
this.polarisStoreService.deleteDatabase(name.getDatabaseName());
88+
this.polarisStoreService.deleteDatabase(name.getCatalogName(), name.getDatabaseName());
8989
} catch (DataIntegrityViolationException exception) {
9090
throw new InvalidMetaException(name, exception);
9191
} catch (Exception exception) {
@@ -101,7 +101,8 @@ public void delete(final ConnectorRequestContext context, final QualifiedName na
101101
public void update(final ConnectorRequestContext context, final DatabaseInfo databaseInfo) {
102102
final QualifiedName name = databaseInfo.getName();
103103
try {
104-
final PolarisDatabaseEntity db = polarisStoreService.getDatabase(name.getDatabaseName())
104+
final PolarisDatabaseEntity db =
105+
polarisStoreService.getDatabase(name.getCatalogName(), name.getDatabaseName())
105106
.orElseThrow(() -> new DatabaseNotFoundException(name));
106107
// currently db objects have no mutable fields so this is noop
107108
db.getAudit().setLastModifiedBy(PolarisUtils.getUserOrDefault(context));
@@ -123,8 +124,9 @@ public void update(final ConnectorRequestContext context, final DatabaseInfo dat
123124
@Override
124125
public DatabaseInfo get(final ConnectorRequestContext context, final QualifiedName name) {
125126
try {
126-
final PolarisDatabaseMapper mapper = new PolarisDatabaseMapper(name.getCatalogName());
127-
final PolarisDatabaseEntity db = polarisStoreService.getDatabase(name.getDatabaseName())
127+
final PolarisDatabaseMapper mapper = new PolarisDatabaseMapper();
128+
final PolarisDatabaseEntity db =
129+
polarisStoreService.getDatabase(name.getCatalogName(), name.getDatabaseName())
128130
.orElseThrow(() -> new DatabaseNotFoundException(name));
129131
return mapper.toInfo(db);
130132
} catch (DatabaseNotFoundException exception) {
@@ -142,7 +144,7 @@ public DatabaseInfo get(final ConnectorRequestContext context, final QualifiedNa
142144
@Override
143145
public boolean exists(final ConnectorRequestContext context, final QualifiedName name) {
144146
try {
145-
return polarisStoreService.getDatabase(name.getDatabaseName()).isPresent();
147+
return polarisStoreService.getDatabase(name.getCatalogName(), name.getDatabaseName()).isPresent();
146148
} catch (Exception exception) {
147149
throw new ConnectorException(
148150
String.format("Failed exists polaris database %s", name), exception);
@@ -163,6 +165,7 @@ public List<QualifiedName> listNames(
163165
try {
164166
final String dbPrefix = prefix == null ? "" : prefix.getDatabaseName();
165167
final List<QualifiedName> qualifiedNames = polarisStoreService.getDatabaseNames(
168+
name.getCatalogName(),
166169
dbPrefix, sort,
167170
connectorContext.getConfig().getListDatabaseNamesPageSize(),
168171
connectorContext.getConfig().isAuroraDataSourceEnabled())
@@ -188,10 +191,11 @@ public List<DatabaseInfo> list(
188191
@Nullable final Pageable pageable
189192
) {
190193
try {
191-
final PolarisDatabaseMapper mapper = new PolarisDatabaseMapper(name.getCatalogName());
194+
final PolarisDatabaseMapper mapper = new PolarisDatabaseMapper();
192195
final String dbPrefix = prefix == null ? "" : prefix.getDatabaseName();
193196

194197
final List<PolarisDatabaseEntity> dbs = polarisStoreService.getDatabases(
198+
name.getCatalogName(),
195199
dbPrefix, sort,
196200
connectorContext.getConfig().getListDatabaseEntitiesPageSize(),
197201
connectorContext.getConfig().isAuroraDataSourceEnabled()

0 commit comments

Comments
 (0)