Skip to content

Commit da66bf8

Browse files
ShenFeng312ShenFeng312
committed
Merge remote-tracking branch 'origin/master'
2 parents bfd3f43 + 8bab753 commit da66bf8

File tree

15 files changed

+475
-54
lines changed

15 files changed

+475
-54
lines changed

‎RELEASE-NOTES.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@
8080
1. SQL Parser: Support Hive SHOW INDEX & SHOW COLUMNS & SHOW FUNCTIONS statement parse - [#36284](https://github.com/apache/shardingsphere/pull/36284)
8181
1. SQL Parser: Support Hive Show Granted Roles and Privileges & SHOW LOCKS & SHOW CONF statement parse - [#36300](https://github.com/apache/shardingsphere/pull/36300)
8282
1. SQL Parser: Support Hive SHOW TRANSACTIONS & SHOW COMPACTIONS statement parse - [#36301](https://github.com/apache/shardingsphere/pull/36301)
83+
1. SQL Parser: Support Hive DESCRIBE statement parse - [#36350](https://github.com/apache/shardingsphere/pull/36350)
8384
1. SQL Parser: Support Hive Inserting data into Hive Tables from queries statement parse - [#36320](https://github.com/apache/shardingsphere/pull/36320)
85+
1. SQL Parser: Support Hive Writing data into the filesystem from queries statement parse - [#36371](https://github.com/apache/shardingsphere/pull/36371)
8486
1. SQL Parser: Support SQL Server xml methods parse - [#35911](https://github.com/apache/shardingsphere/pull/35911)
8587
1. SQL Parser: Support SQL Server CHANGETABLE function parse - [#35920](https://github.com/apache/shardingsphere/pull/35920)
8688
1. SQL Parser: Support SQL Server AI_GENERATE_EMBEDDINGS function parse - [#35922](https://github.com/apache/shardingsphere/pull/35922)

‎parser/sql/dialect/hive/src/main/antlr4/imports/hive/BaseRule.g4‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,10 @@ columnName
648648
: identifier
649649
;
650650

651+
connectorName
652+
: identifier
653+
;
654+
651655
indexName
652656
: identifier
653657
;
@@ -1332,3 +1336,48 @@ noWriteToBinLog
13321336
channelOption
13331337
: FOR CHANNEL string_
13341338
;
1339+
1340+
propertyListCommonClause
1341+
: LP_ propertyList RP_
1342+
;
1343+
1344+
propertyList
1345+
: property (COMMA_ property)*
1346+
;
1347+
1348+
property
1349+
: string_ EQ_ string_
1350+
;
1351+
1352+
rowFormat
1353+
: ROW FORMAT rowFormatType
1354+
;
1355+
1356+
rowFormatType
1357+
: DELIMITED rowFormatDelimited
1358+
| SERDE string_ (WITH SERDEPROPERTIES propertyListCommonClause)?
1359+
;
1360+
1361+
rowFormatDelimited
1362+
: (COLUMNS TERMINATED BY string_ (ESCAPED BY string_)?)?
1363+
(COLLECTION ITEMS TERMINATED BY string_)?
1364+
(MAP KEYS TERMINATED BY string_)?
1365+
(LINES TERMINATED BY string_)?
1366+
(NULL DEFINED AS string_)?
1367+
;
1368+
1369+
storedClause
1370+
: STORED AS fileFormat
1371+
| STORED BY string_ (WITH SERDEPROPERTIES propertyListCommonClause)?
1372+
;
1373+
1374+
fileFormat
1375+
: SEQUENCEFILE
1376+
| TEXTFILE
1377+
| RCFILE
1378+
| ORC
1379+
| PARQUET
1380+
| AVRO
1381+
| JSONFILE
1382+
| INPUTFORMAT string_ OUTPUTFORMAT string_
1383+
;

‎parser/sql/dialect/hive/src/main/antlr4/imports/hive/DALStatement.g4‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@ show
3939
| showCompactions
4040
;
4141

42+
describe
43+
: describeDatabase
44+
| describeConnector
45+
| describeTable
46+
;
47+
4248
showDatabases
4349
: SHOW (DATABASES|SCHEMAS) showLike?
4450
;
@@ -125,3 +131,28 @@ showFrom
125131
showLike
126132
: LIKE stringLiterals
127133
;
134+
135+
describeDatabase
136+
: DESCRIBE (DATABASE | SCHEMA) EXTENDED? databaseName
137+
;
138+
139+
describeConnector
140+
: DESCRIBE CONNECTOR EXTENDED? connectorName
141+
;
142+
143+
describeTable
144+
: DESCRIBE (EXTENDED | FORMATTED)? tableName partitionSpec? columnClause?
145+
;
146+
147+
columnClause
148+
: columnName columnOptions
149+
;
150+
151+
columnOptions
152+
: columnOption*
153+
;
154+
155+
columnOption
156+
: DOT_ identifier
157+
| DOT_ string_
158+
;

‎parser/sql/dialect/hive/src/main/antlr4/imports/hive/DDLStatement.g4‎

Lines changed: 0 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -338,39 +338,6 @@ skewedValue
338338
| literals
339339
;
340340

341-
rowFormat
342-
: ROW FORMAT rowFormatType
343-
;
344-
345-
rowFormatType
346-
: DELIMITED rowFormatDelimited
347-
| SERDE string_ (WITH SERDEPROPERTIES propertyListCommonClause)?
348-
;
349-
350-
rowFormatDelimited
351-
: (COLUMNS TERMINATED BY string_ (ESCAPED BY string_)?)?
352-
(COLLECTION ITEMS TERMINATED BY string_)?
353-
(MAP KEYS TERMINATED BY string_)?
354-
(LINES TERMINATED BY string_)?
355-
(NULL DEFINED AS string_)?
356-
;
357-
358-
storedClause
359-
: STORED AS fileFormat
360-
| STORED BY string_ (WITH SERDEPROPERTIES propertyListCommonClause)?
361-
;
362-
363-
fileFormat
364-
: SEQUENCEFILE
365-
| TEXTFILE
366-
| RCFILE
367-
| ORC
368-
| PARQUET
369-
| AVRO
370-
| JSONFILE
371-
| INPUTFORMAT string_ OUTPUTFORMAT string_
372-
;
373-
374341
storageLocation
375342
: LOCATION string_
376343
;
@@ -387,14 +354,6 @@ tblProperties
387354
: TBLPROPERTIES propertyListCommonClause
388355
;
389356

390-
propertyList
391-
: property (COMMA_ property)*
392-
;
393-
394-
property
395-
: string_ EQ_ string_
396-
;
397-
398357
addConstraint
399358
: ADD CONSTRAINT constraintName
400359
;
@@ -495,10 +454,6 @@ columnNamesCommonClause
495454
: LP_ columnNames RP_
496455
;
497456

498-
propertyListCommonClause
499-
: LP_ propertyList RP_
500-
;
501-
502457
macroParameterList
503458
: macroParameter (COMMA_ macroParameter)*
504459
;

‎parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import BaseRule;
2222
insert
2323
: INSERT insertSpecification INTO? tableName partitionNames? (insertValuesClause | setAssignmentsClause | insertSelectClause) onDuplicateKeyClause?
2424
| insertDataIntoTablesFromQueries
25+
| writingDataIntoFileSystem
2526
;
2627

2728
insertSpecification
@@ -368,7 +369,7 @@ dynamicPartitionInserts
368369
;
369370

370371
hiveMultipleInserts
371-
: hiveInsertStatement (hiveInsertStatement)*
372+
: hiveInsertStatement+
372373
;
373374

374375
hiveInsertStatement
@@ -387,3 +388,12 @@ dynamicPartitionKey
387388
partitionClause
388389
: partitionSpec ifNotExists?
389390
;
391+
392+
writingDataIntoFileSystem
393+
: insertOverwriteStandardSyntax
394+
| fromClause insertOverwriteStandardSyntax+
395+
;
396+
397+
insertOverwriteStandardSyntax
398+
: INSERT OVERWRITE LOCAL? DIRECTORY string_ rowFormat? storedClause? select
399+
;

‎parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3374,6 +3374,10 @@ JAR
33743374
: J A R
33753375
;
33763376

3377+
CONNECTOR
3378+
: C O N N E C T O R
3379+
;
3380+
33773381
CONNECTORS
33783382
: C O N N E C T O R S
33793383
;

‎parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ execute
5050
| dropFunction
5151
| reloadFunction
5252
| show
53+
| describe
5354
) (SEMI_ EOF? | EOF)
5455
| EOF
5556
;

‎parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDALStatementVisitor.java‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.UseContext;
2424
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowDatabasesContext;
2525
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowLikeContext;
26+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowFromContext;
2627
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowConnectorsContext;
2728
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowTablesContext;
2829
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowViewsContext;
@@ -39,10 +40,14 @@
3940
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowConfContext;
4041
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowTransactionsContext;
4142
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.ShowCompactionsContext;
43+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DescribeDatabaseContext;
44+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DescribeConnectorContext;
45+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DescribeTableContext;
4246
import org.apache.shardingsphere.sql.parser.hive.visitor.statement.HiveStatementVisitor;
4347
import org.apache.shardingsphere.sql.parser.statement.core.segment.dal.FromDatabaseSegment;
4448
import org.apache.shardingsphere.sql.parser.statement.core.segment.dal.ShowFilterSegment;
4549
import org.apache.shardingsphere.sql.parser.statement.core.segment.dal.ShowLikeSegment;
50+
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.column.ColumnSegment;
4651
import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.DatabaseSegment;
4752
import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
4853
import org.apache.shardingsphere.sql.parser.statement.core.value.identifier.IdentifierValue;
@@ -59,12 +64,14 @@
5964
import org.apache.shardingsphere.sql.parser.statement.hive.dal.show.HiveShowTransactionsStatement;
6065
import org.apache.shardingsphere.sql.parser.statement.hive.dal.show.HiveShowCompactionsStatement;
6166
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.MySQLUseStatement;
67+
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.column.MySQLDescribeStatement;
6268
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.database.MySQLShowDatabasesStatement;
6369
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.privilege.MySQLShowGrantsStatement;
6470
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.table.MySQLShowCreateTableStatement;
6571
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.table.MySQLShowTablesStatement;
6672
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.index.MySQLShowIndexStatement;
6773
import org.apache.shardingsphere.sql.parser.statement.mysql.dal.show.column.MySQLShowColumnsStatement;
74+
import org.apache.shardingsphere.sql.parser.statement.hive.dal.HiveDescribeStatement;
6875

6976
/**
7077
* DAL statement visitor for Hive.
@@ -157,14 +164,19 @@ public ASTNode visitShowCreateTable(final ShowCreateTableContext ctx) {
157164
public ASTNode visitShowIndex(final ShowIndexContext ctx) {
158165
FromDatabaseSegment fromDatabase = null;
159166
if (null != ctx.showFrom()) {
160-
ASTNode showFromNode = visit(ctx.showFrom());
161-
if (showFromNode instanceof DatabaseSegment) {
162-
fromDatabase = new FromDatabaseSegment(ctx.showFrom().getStart().getStartIndex(), (DatabaseSegment) showFromNode);
163-
}
167+
fromDatabase = createFromDatabaseSegment(ctx.showFrom());
164168
}
165169
return new MySQLShowIndexStatement(getDatabaseType(), (SimpleTableSegment) visit(ctx.tableName()), fromDatabase);
166170
}
167171

172+
private FromDatabaseSegment createFromDatabaseSegment(final ShowFromContext showFromContext) {
173+
ASTNode showFromNode = visit(showFromContext);
174+
if (showFromNode instanceof DatabaseSegment) {
175+
return new FromDatabaseSegment(showFromContext.getStart().getStartIndex(), (DatabaseSegment) showFromNode);
176+
}
177+
return null;
178+
}
179+
168180
@Override
169181
public ASTNode visitShowColumns(final ShowColumnsContext ctx) {
170182
SimpleTableSegment table = null;
@@ -173,10 +185,7 @@ public ASTNode visitShowColumns(final ShowColumnsContext ctx) {
173185
}
174186
FromDatabaseSegment fromDatabase = null;
175187
if (null != ctx.showFrom()) {
176-
ASTNode showFromNode = visit(ctx.showFrom());
177-
if (showFromNode instanceof DatabaseSegment) {
178-
fromDatabase = new FromDatabaseSegment(ctx.showFrom().getStart().getStartIndex(), (DatabaseSegment) showFromNode);
179-
}
188+
fromDatabase = createFromDatabaseSegment(ctx.showFrom());
180189
}
181190
ShowFilterSegment filter = null;
182191
if (null != ctx.showLike()) {
@@ -215,4 +224,24 @@ public ASTNode visitShowTransactions(final ShowTransactionsContext ctx) {
215224
public ASTNode visitShowCompactions(final ShowCompactionsContext ctx) {
216225
return new HiveShowCompactionsStatement(getDatabaseType());
217226
}
227+
228+
@Override
229+
public ASTNode visitDescribeDatabase(final DescribeDatabaseContext ctx) {
230+
return new HiveDescribeStatement(getDatabaseType());
231+
}
232+
233+
@Override
234+
public ASTNode visitDescribeConnector(final DescribeConnectorContext ctx) {
235+
return new HiveDescribeStatement(getDatabaseType());
236+
}
237+
238+
@Override
239+
public ASTNode visitDescribeTable(final DescribeTableContext ctx) {
240+
SimpleTableSegment table = (SimpleTableSegment) visit(ctx.tableName());
241+
ColumnSegment columnWildcard = null;
242+
if (null != ctx.columnClause()) {
243+
columnWildcard = (ColumnSegment) visit(ctx.columnClause().columnName());
244+
}
245+
return new MySQLDescribeStatement(getDatabaseType(), table, columnWildcard);
246+
}
218247
}

‎parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@
123123
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DynamicPartitionInsertsContext;
124124
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveMultipleInsertsContext;
125125
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveInsertStatementContext;
126+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.WritingDataIntoFileSystemContext;
127+
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertOverwriteStandardSyntaxContext;
126128
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DynamicPartitionClauseContext;
127129
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.DynamicPartitionKeyContext;
128130
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TableNameContext;
@@ -873,6 +875,9 @@ public ASTNode visitInsert(final InsertContext ctx) {
873875
if (null != ctx.insertDataIntoTablesFromQueries()) {
874876
return visit(ctx.insertDataIntoTablesFromQueries());
875877
}
878+
if (null != ctx.writingDataIntoFileSystem()) {
879+
return visit(ctx.writingDataIntoFileSystem());
880+
}
876881
InsertStatement result;
877882
if (null != ctx.insertValuesClause()) {
878883
result = (InsertStatement) visit(ctx.insertValuesClause());
@@ -990,6 +995,51 @@ public ASTNode visitDynamicPartitionKey(final DynamicPartitionKeyContext ctx) {
990995
return new PartitionSegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), new IdentifierValue(ctx.identifier().getText()));
991996
}
992997

998+
@Override
999+
public ASTNode visitWritingDataIntoFileSystem(final WritingDataIntoFileSystemContext ctx) {
1000+
List<InsertOverwriteStandardSyntaxContext> statements = ctx.insertOverwriteStandardSyntax();
1001+
if (1 == statements.size() && null == ctx.fromClause()) {
1002+
return visit(statements.get(0));
1003+
}
1004+
final TableSegment sourceTable = null != ctx.fromClause() ? (TableSegment) visit(ctx.fromClause()) : null;
1005+
if (1 == statements.size()) {
1006+
InsertStatement single = (InsertStatement) visit(statements.get(0));
1007+
if (null != sourceTable) {
1008+
single.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
1009+
}
1010+
single.addParameterMarkers(getParameterMarkerSegments());
1011+
return single;
1012+
}
1013+
InsertStatement result = new InsertStatement(getDatabaseType());
1014+
result.setMultiTableInsertType(MultiTableInsertType.ALL);
1015+
MultiTableInsertIntoSegment multiTableInsertInto = new MultiTableInsertIntoSegment(
1016+
ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex());
1017+
for (InsertOverwriteStandardSyntaxContext each : statements) {
1018+
InsertStatement insertStmt = (InsertStatement) visit(each);
1019+
if (null != sourceTable) {
1020+
insertStmt.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
1021+
}
1022+
insertStmt.addParameterMarkers(getParameterMarkerSegments());
1023+
multiTableInsertInto.getInsertStatements().add(insertStmt);
1024+
}
1025+
result.setMultiTableInsertInto(multiTableInsertInto);
1026+
result.addParameterMarkers(getParameterMarkerSegments());
1027+
return result;
1028+
}
1029+
1030+
@Override
1031+
public ASTNode visitInsertOverwriteStandardSyntax(final InsertOverwriteStandardSyntaxContext ctx) {
1032+
return createHiveInsertStatementForDirectory(ctx.select(), ctx.start.getStartIndex());
1033+
}
1034+
1035+
private InsertStatement createHiveInsertStatementForDirectory(final SelectContext select, final int startIndex) {
1036+
InsertStatement result = new InsertStatement(getDatabaseType());
1037+
result.setInsertColumns(new InsertColumnsSegment(startIndex, startIndex, Collections.emptyList()));
1038+
result.setInsertSelect(createInsertSelectSegment(select));
1039+
result.addParameterMarkers(getParameterMarkerSegments());
1040+
return result;
1041+
}
1042+
9931043
private SubquerySegment createInsertSelectSegment(final SelectContext ctx) {
9941044
SelectStatement selectStatement = (SelectStatement) visit(ctx);
9951045
return new SubquerySegment(ctx.start.getStartIndex(), ctx.stop.getStopIndex(), selectStatement, getOriginalText(ctx));

0 commit comments

Comments
 (0)