MySQL WITH clause recursive subquery error fix

RELEASE-NOTES.md

@@ -21,6 +21,7 @@
 1. SQL Binder: Support column definition for the WITH clause and ExternalTableBinderContext in CommonTableExpressionBinder.[#34384](https://github.com/apache/shardingsphere/pull/34384)
 1. SQL Binder: Support case when then else segment bind - [#34600](https://github.com/apache/shardingsphere/pull/34600)
 1. SQL Binder: Support outer join expression bind - [#35019](https://github.com/apache/shardingsphere/pull/35019)
+1. SQL Binder: Support ShorthandProjections while using RECURSIVE in WITH clause in CommonTableExpressionBinder. [#35146](https://github.com/apache/shardingsphere/pull/35146)
 1. SQL Parser: Support MySQL SELECT CAST AS YEAR statement parse - [#34638](https://github.com/apache/shardingsphere/pull/34638)
 1. SQL Parser: Support MySQL SELECT MAX(ALL expr) statement parse - [#34639](https://github.com/apache/shardingsphere/pull/34639)
 1. SQL Parser: Support MySQL INSERT with GEOMCOLLECTION function parse - [#34654](https://github.com/apache/shardingsphere/pull/34654)

docs/document/content/user-manual/error-code/sql-error-code.cn.md

@@ -73,6 +73,7 @@ SQL 错误码以标准的 SQL State，Vendor Code 和详细错误信息提供，
 | 12300       | 0A000     | DROP TABLE ... CASCADE is not supported.                                                                                         |
 | 12500       | 42000     | Not unique table/alias: '%s'.                                                                                                    |
 | 12600       | HY000     | In definition of view, derived table or common table expression, SELECT list and column names list have different column counts. |
+| 12700       | HY000     | Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones.|
 
 ### 连接
 

docs/document/content/user-manual/error-code/sql-error-code.en.md

@@ -73,6 +73,7 @@ SQL error codes provide by standard `SQL State`, `Vendor Code` and `Reason`, whi
 | 12300       | 0A000     | DROP TABLE ... CASCADE is not supported.                                                                                         |
 | 12500       | 42000     | Not unique table/alias: '%s'.                                                                                                    |
 | 12600       | HY000     | In definition of view, derived table or common table expression, SELECT list and column names list have different column counts. |
+| 12700       | HY000     | Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones.|
 
 ### Connection
 

infra/binder/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/context/type/SimpleTableSegmentBinderContext.java

@@ -18,18 +18,25 @@
 package org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.type;
 
 import com.cedarsoftware.util.CaseInsensitiveMap;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.TableSegmentBinderContext;
+import org.apache.shardingsphere.infra.binder.engine.statement.SQLStatementBinderContext;
+import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import org.apache.shardingsphere.infra.database.core.type.DatabaseTypeRegistry;
+import org.apache.shardingsphere.infra.database.opengauss.type.OpenGaussDatabaseType;
+import org.apache.shardingsphere.infra.database.postgresql.type.PostgreSQLDatabaseType;
+import org.apache.shardingsphere.infra.metadata.database.schema.manager.SystemSchemaManager;
 import org.apache.shardingsphere.sql.parser.statement.core.enums.TableSourceType;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ShorthandProjectionSegment;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.Optional;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.value.identifier.IdentifierValue;
 
 /**
  * Simple table segment binder context.
@@ -39,6 +46,8 @@
 @Setter
 public final class SimpleTableSegmentBinderContext implements TableSegmentBinderContext {
 
+    private static final String PG_CATALOG = "pg_catalog";
+
     @Getter(AccessLevel.NONE)
     private final Map<String, ProjectionSegment> columnLabelProjectionSegments;
 
@@ -69,4 +78,24 @@ public Optional<ProjectionSegment> findProjectionSegmentByColumnLabel(final Stri
     public Collection<ProjectionSegment> getProjectionSegments() {
         return columnLabelProjectionSegments.values();
     }
+
+    /**
+     * Get schema name.
+     *
+     * @param segment simple table segment
+     * @param binderContext statement binder context
+     * @return schema identifier value
+     */
+    public static IdentifierValue getSchemaName(final SimpleTableSegment segment, final SQLStatementBinderContext binderContext) {
+        if (segment.getOwner().isPresent()) {
+            return segment.getOwner().get().getIdentifier();
+        }
+        // TODO getSchemaName according to search path
+        DatabaseType databaseType = binderContext.getSqlStatement().getDatabaseType();
+        if ((databaseType instanceof PostgreSQLDatabaseType || databaseType instanceof OpenGaussDatabaseType)
+                && SystemSchemaManager.isSystemTable(databaseType.getType(), PG_CATALOG, segment.getTableName().getIdentifier().getValue())) {
+            return new IdentifierValue(PG_CATALOG);
+        }
+        return new IdentifierValue(new DatabaseTypeRegistry(databaseType).getDefaultSchemaName(binderContext.getCurrentDatabaseName()));
+    }
 }

infra/binder/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinder.java

@@ -27,7 +27,6 @@
 import org.apache.shardingsphere.infra.binder.engine.statement.SQLStatementBinderContext;
 import org.apache.shardingsphere.infra.database.core.metadata.database.enums.QuoteCharacter;
 import org.apache.shardingsphere.infra.database.core.metadata.database.metadata.DialectDatabaseMetaData;
-import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
 import org.apache.shardingsphere.infra.database.core.type.DatabaseTypeRegistry;
 import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions;
 import org.apache.shardingsphere.infra.exception.dialect.exception.syntax.database.NoDatabaseSelectedException;
@@ -111,25 +110,12 @@ private static IdentifierValue getDatabaseName(final SimpleTableSegment segment,
     }
 
     private static IdentifierValue getSchemaName(final SimpleTableSegment segment, final SQLStatementBinderContext binderContext, final IdentifierValue databaseName) {
-        IdentifierValue result = getSchemaName(segment, binderContext);
+        IdentifierValue result = SimpleTableSegmentBinderContext.getSchemaName(segment, binderContext);
         ShardingSpherePreconditions.checkState(binderContext.getMetaData().getDatabase(databaseName.getValue()).containsSchema(result.getValue()),
                 () -> new SchemaNotFoundException(result.getValue()));
         return result;
     }
 
-    private static IdentifierValue getSchemaName(final SimpleTableSegment segment, final SQLStatementBinderContext binderContext) {
-        if (segment.getOwner().isPresent()) {
-            return segment.getOwner().get().getIdentifier();
-        }
-        // TODO getSchemaName according to search path
-        DatabaseType databaseType = binderContext.getSqlStatement().getDatabaseType();
-        DatabaseTypeRegistry databaseTypeRegistry = new DatabaseTypeRegistry(databaseType);
-        Optional<String> defaultSystemSchema = databaseTypeRegistry.getDialectDatabaseMetaData().getSchemaOption().getDefaultSystemSchema();
-        return defaultSystemSchema.isPresent() && SystemSchemaManager.isSystemTable(databaseType.getType(), defaultSystemSchema.get(), segment.getTableName().getIdentifier().getValue())
-                ? new IdentifierValue(defaultSystemSchema.get())
-                : new IdentifierValue(databaseTypeRegistry.getDefaultSchemaName(binderContext.getCurrentDatabaseName()));
-    }
-
     private static void checkTableExists(final SQLStatementBinderContext binderContext, final ShardingSphereSchema schema, final String schemaName, final String tableName) {
         // TODO refactor table exists check with spi @duanzhengqiang
         if (binderContext.getSqlStatement() instanceof CreateTableStatement && isCreateTable(((CreateTableStatement) binderContext.getSqlStatement()).getTable(), tableName)) {

infra/binder/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/with/CommonTableExpressionSegmentBinder.java

@@ -31,21 +31,25 @@
 import org.apache.shardingsphere.infra.binder.engine.segment.util.SubqueryTableBindUtils;
 import org.apache.shardingsphere.infra.binder.engine.statement.SQLStatementBinderContext;
 import org.apache.shardingsphere.infra.exception.core.ShardingSpherePreconditions;
+import org.apache.shardingsphere.infra.exception.kernel.syntax.CommonTableExpressionRecursiveSubQueryRequiresNonRecursiveBlockFirstException;
 import org.apache.shardingsphere.infra.exception.kernel.syntax.DifferenceInColumnCountOfSelectListAndColumnNameListException;
 import org.apache.shardingsphere.infra.exception.kernel.syntax.DuplicateCommonTableExpressionAliasException;
 import org.apache.shardingsphere.sql.parser.statement.core.enums.TableSourceType;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.column.ColumnSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.complex.CommonTableExpressionSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ColumnProjectionSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ShorthandProjectionSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.bound.ColumnSegmentBoundInfo;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.bound.TableSegmentBoundInfo;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SubqueryTableSegment;
 import org.apache.shardingsphere.sql.parser.statement.core.statement.dml.SelectStatement;
 import org.apache.shardingsphere.sql.parser.statement.core.value.identifier.IdentifierValue;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
@@ -75,7 +79,10 @@ public static CommonTableExpressionSegment bind(final CommonTableExpressionSegme
         }
         if (recursive && segment.getAliasName().isPresent()) {
             binderContext.getExternalTableBinderContexts().put(new CaseInsensitiveString(segment.getAliasName().get()),
-                    new SimpleTableSegmentBinderContext(segment.getColumns().stream().map(ColumnProjectionSegment::new).collect(Collectors.toList()), TableSourceType.TEMPORARY_TABLE));
+                    new SimpleTableSegmentBinderContext(
+                            createProjectionSegmentForRecursiveCommonTableExpression(segment.getColumns().stream().map(ColumnProjectionSegment::new).collect(Collectors.toList()), binderContext,
+                                    segment.getSubquery().getSelect(), segment.getAliasName().get()),
+                            TableSourceType.TEMPORARY_TABLE));
         }
         SubqueryTableSegment subqueryTableSegment = new SubqueryTableSegment(segment.getStartIndex(), segment.getStopIndex(), segment.getSubquery());
         subqueryTableSegment.setAlias(segment.getAliasSegment());
@@ -91,6 +98,34 @@ public static CommonTableExpressionSegment bind(final CommonTableExpressionSegme
         return result;
     }
 
+    private static Collection<ProjectionSegment> createProjectionSegmentForRecursiveCommonTableExpression(final Collection<ProjectionSegment> definitionColumns,
+                                                                                                          final SQLStatementBinderContext binderContext,
+                                                                                                          final SelectStatement select, final String cteAlias) {
+        if (select.getFrom().isPresent()) {
+            ShardingSpherePreconditions.checkState(!((SimpleTableSegment) select.getFrom().get()).getTableName().getIdentifier().getValue().equals(cteAlias),
+                    () -> new CommonTableExpressionRecursiveSubQueryRequiresNonRecursiveBlockFirstException(cteAlias));
+        }
+
+        Collection<ProjectionSegment> subqueryProjections = new LinkedList<>();
+        for (ProjectionSegment each : select.getProjections().getProjections()) {
+            if (each instanceof ShorthandProjectionSegment && select.getFrom().isPresent()) {
+                SimpleTableSegment simpleTableSegment = (SimpleTableSegment) select.getFrom().get();
+
+                IdentifierValue schema = SimpleTableSegmentBinderContext.getSchemaName(simpleTableSegment, binderContext);
+                Collection<ProjectionSegment> projectionSegments = binderContext.getMetaData().getDatabase(binderContext.getCurrentDatabaseName()).getSchema(schema.getValue())
+                        .getVisibleColumnNames(simpleTableSegment.getTableName().getIdentifier().getValue()).stream()
+                        .map(col -> new ColumnProjectionSegment(new ColumnSegment(0, 0, new IdentifierValue(col)))).collect(Collectors.toList());
+                subqueryProjections.addAll(SubqueryTableBindUtils.createSubqueryProjections(projectionSegments, new IdentifierValue(""),
+                        binderContext.getSqlStatement().getDatabaseType(), TableSourceType.TEMPORARY_TABLE));
+            } else {
+                subqueryProjections.addAll(SubqueryTableBindUtils.createSubqueryProjections(Collections.singleton(each), new IdentifierValue(""),
+                        binderContext.getSqlStatement().getDatabaseType(), TableSourceType.TEMPORARY_TABLE));
+            }
+        }
+
+        return definitionColumns.isEmpty() ? subqueryProjections : definitionColumns;
+    }
+
     private static Multimap<CaseInsensitiveString, TableSegmentBinderContext> createCurrentTableBinderContexts(final Collection<ColumnSegment> definitionColumns,
                                                                                                                final SQLStatementBinderContext binderContext, final SelectStatement selectStatement) {
         Collection<ProjectionSegment> subqueryProjections = SubqueryTableBindUtils.createSubqueryProjections(selectStatement.getProjections().getProjections(), new IdentifierValue(""),

infra/common/src/main/java/org/apache/shardingsphere/infra/exception/kernel/syntax/CommonTableExpressionRecursiveSubQueryRequiresNonRecursiveBlockFirstException.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.infra.exception.kernel.syntax;
+
+import org.apache.shardingsphere.infra.exception.core.external.sql.sqlstate.XOpenSQLState;
+import org.apache.shardingsphere.infra.exception.core.external.sql.type.kernel.category.SyntaxSQLException;
+
+/**
+ * Common table expression recursive requires non recursive first exception.
+ */
+public final class CommonTableExpressionRecursiveSubQueryRequiresNonRecursiveBlockFirstException extends SyntaxSQLException {
+
+    private static final long serialVersionUID = -6759214908809202572L;
+
+    public CommonTableExpressionRecursiveSubQueryRequiresNonRecursiveBlockFirstException(final String alias) {
+        super(XOpenSQLState.GENERAL_ERROR, 700, "Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones", alias);
+    }
+}

test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-using-with-clause.xml

@@ -23,4 +23,12 @@
     <test-case sql="WITH products (col1, col2, col3, col4, col5, col6) AS (SELECT * FROM t_product) SELECT col1 FROM products" db-types="MySQL" scenario-types="db">
         <assertion />
     </test-case>
+
+    <test-case sql="WITH RECURSIVE cte AS (SELECT * from t_user UNION ALL SELECT user_id+1, user_name, password, email, telephone, creation_date FROM cte WHERE user_id BETWEEN 1 AND 4) SELECT * FROM cte" db-types="MySQL" scenario-types="db">
+        <assertion />
+    </test-case>
+
+    <test-case sql="WITH RECURSIVE cte AS (SELECT *,user_name as repeat_name from t_user UNION ALL SELECT user_id+1, user_name, password, email, telephone, creation_date, repeat_name FROM cte WHERE user_id BETWEEN 1 AND 4) SELECT user_id FROM cte" db-types="MySQL" scenario-types="db">
+        <assertion />
+    </test-case>
 </e2e-test-cases>

test/it/parser/src/main/resources/case/dml/select-with.xml

@@ -512,7 +512,7 @@
         </from>
     </select>
 
-    <select sql-case-id="select_with_recursive_union_all1">
+    <select sql-case-id="select_with_recursive_union_all">
         <with start-index="0" stop-index="217">
             <common-table-expression name="DirectoryCTE" start-index="15" stop-index="217">
                 <subquery-expression start-index="15" stop-index="217">
@@ -626,7 +626,8 @@
             <column-item name="level" order-direction="ASC" start-index="255" stop-index="259" />
         </order-by>
     </select>
-    <select sql-case-id="select_with_oracle_recursive_union_all1">
+
+    <select sql-case-id="select_with_oracle_union_all">
         <with start-index="0" stop-index="243">
             <common-table-expression name="DirectoryCTE" start-index="5" stop-index="243">
                 <subquery-expression start-index="22" stop-index="242">
@@ -762,7 +763,8 @@
             <column-item name="level" order-direction="ASC" start-index="316" stop-index="320"/>
         </order-by>
     </select>
-    <select sql-case-id="select_with_recursive_union_all2">
+
+    <select sql-case-id="select_with_union_all_subquery">
         <with start-index="0" stop-index="62">
             <common-table-expression name="cte" start-index="5" stop-index="62">
                 <subquery-expression start-index="5" stop-index="62">