Skip to content

Commit 03fd0fa

Browse files
stephen-shelbystephen-shelby
committed
xxxx
Signed-off-by: stephen <[email protected]>
1 parent 7d81781 commit 03fd0fa

File tree

12 files changed

+108
-26
lines changed

12 files changed

+108
-26
lines changed

‎fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,8 @@ public static MaterializedViewRewriteMode parse(String str) {
782782

783783
public static final String ENABLE_PLAN_VALIDATION = "enable_plan_validation";
784784

785+
public static final String ENABLE_OPTIMIZER_RULE_DEBUG = "enable_optimizer_rule_debug";
786+
785787
public static final String ENABLE_STRICT_TYPE = "enable_strict_type";
786788

787789
public static final String PARTIAL_UPDATE_MODE = "partial_update_mode";
@@ -2463,6 +2465,9 @@ public long getConnectorSinkTargetMaxFileSize() {
24632465
@VarAttr(name = ENABLE_PLAN_VALIDATION, flag = VariableMgr.INVISIBLE)
24642466
private boolean enablePlanValidation = true;
24652467

2468+
@VarAttr(name = ENABLE_OPTIMIZER_RULE_DEBUG)
2469+
private boolean enableOptimizerRuleDebug = true;
2470+
24662471
@VarAttr(name = SCAN_OR_TO_UNION_LIMIT, flag = VariableMgr.INVISIBLE)
24672472
private int scanOrToUnionLimit = 4;
24682473

@@ -3703,7 +3708,7 @@ public long getOptimizerExecuteTimeout() {
37033708
}
37043709

37053710
public void setOptimizerExecuteTimeout(long optimizerExecuteTimeout) {
3706-
this.optimizerExecuteTimeout = optimizerExecuteTimeout;
3711+
this.optimizerExecuteTimeout = 100000;
37073712
}
37083713

37093714
public QueryDebugOptions getQueryDebugOptions() {
@@ -4731,6 +4736,10 @@ public void setEnablePlanValidation(boolean val) {
47314736
this.enablePlanValidation = val;
47324737
}
47334738

4739+
public boolean enableOptimizerRuleDebug() {
4740+
return this.enableOptimizerRuleDebug;
4741+
}
4742+
47344743
public boolean isCboPruneSubfield() {
47354744
return cboPruneSubfield;
47364745
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@
131131
import com.starrocks.sql.optimizer.task.TaskScheduler;
132132
import com.starrocks.sql.optimizer.validate.MVRewriteValidator;
133133
import com.starrocks.sql.optimizer.validate.OptExpressionValidator;
134-
import com.starrocks.sql.optimizer.validate.PlanValidator;
135134
import org.apache.commons.collections4.CollectionUtils;
136135
import org.apache.logging.log4j.LogManager;
137136
import org.apache.logging.log4j.Logger;
@@ -309,8 +308,10 @@ private OptExpression optimizeByCost(ConnectContext connectContext,
309308
mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add);
310309

311310
try (Timer ignored = Tracers.watchScope("PlanValidate")) {
311+
// Enable all checkers for final validation
312+
rootTaskContext.getPlanValidator().enableAllCheckers();
312313
// valid the final plan
313-
PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext);
314+
rootTaskContext.getPlanValidator().validatePlan(finalPlan, rootTaskContext);
314315
// validate mv and log tracer if needed
315316
MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators);
316317
mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext);
@@ -588,6 +589,7 @@ private OptExpression logicalRuleRewrite(
588589

589590
rootTaskContext.setRequiredColumns(requiredColumns.clone());
590591
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
592+
rootTaskContext.getPlanValidator().enableInputDependenciesChecker();
591593

592594
pruneTables(tree, rootTaskContext, requiredColumns);
593595

@@ -663,7 +665,6 @@ private OptExpression logicalRuleRewrite(
663665

664666
scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE);
665667
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
666-
667668
scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule());
668669
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
669670
scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule());
@@ -699,10 +700,8 @@ private OptExpression logicalRuleRewrite(
699700
scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule());
700701
// before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction
701702
scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance());
702-
703703
// After this rule, we shouldn't generate logical project operator
704704
scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
705-
706705
scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule());
707706
scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule());
708707
// intersect rewrite depend on statistics
@@ -720,7 +719,6 @@ private OptExpression logicalRuleRewrite(
720719
scheduler.rewriteOnce(tree, rootTaskContext, JsonPathRewriteRule.createForOlapScan());
721720
scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMinMaxByMonotonicFunctionRule());
722721
scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT);
723-
724722
// NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle
725723
// select count(distinct c) from t group by a, b
726724
// if this rule has applied before MV.
@@ -975,7 +973,6 @@ void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTask
975973
private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
976974
OptExpression result) {
977975
Preconditions.checkState(result.getOp().isPhysical());
978-
979976
int planCount = result.getPlanCount();
980977

981978
// Since there may be many different plans in the logic phase, it's possible

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/RowOutputInfo.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ public List<ColumnOutputInfo> getCommonColInfo() {
137137
}
138138

139139
public List<ColumnRefOperator> getOutputColRefs() {
140-
return chooseOutputMap().values().stream().map(e -> e.getColumnRef()).collect(Collectors.toList());
140+
return chooseOutputMap().values().stream().map(ColumnOutputInfo::getColumnRef).collect(Collectors.toList());
141141
}
142142

143143
public Map<ColumnRefOperator, ScalarOperator> getColumnRefMap() {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/ColumnRefSet.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public ColumnRefSet(Collection<ColumnRefOperator> refs) {
4747

4848
public static ColumnRefSet createByIds(Collection<Integer> colIds) {
4949
ColumnRefSet columnRefSet = new ColumnRefSet();
50-
colIds.stream().forEach(columnRefSet::union);
50+
colIds.forEach(columnRefSet::union);
5151
return columnRefSet;
5252
}
5353

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggregateGroupingSetsRule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,4 +357,4 @@ private OptExpression buildSubRepeatConsume(ColumnRefFactory factory,
357357
return OptExpression.create(projectOperator,
358358
OptExpression.create(newAggregate, OptExpression.create(newRepeat, OptExpression.create(consume))));
359359
}
360-
}
360+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public void execute() {
8989
}
9090
List<OptExpression> targetExpressions;
9191
OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, extractExpr);
92+
9293
try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.getClass().getSimpleName())) {
9394
targetExpressions = rule.transform(extractExpr, context.getOptimizerContext());
9495
} catch (StarRocksPlannerException e) {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.starrocks.sql.optimizer.operator.OperatorType;
2424
import com.starrocks.sql.optimizer.operator.pattern.Pattern;
2525
import com.starrocks.sql.optimizer.rule.Rule;
26+
import com.starrocks.sql.optimizer.validate.PlanValidator;
2627

2728
import java.util.List;
2829

@@ -62,8 +63,22 @@ public void execute() {
6263
.getOptimizerOptions().isRuleDisable(rule.type()))) {
6364
return;
6465
}
66+
67+
// Save whole tree state before rule group execution for validation
68+
OptExpression wholePlanBefore = null;
69+
if (context.getOptimizerContext().getSessionVariable().enableOptimizerRuleDebug()) {
70+
wholePlanBefore = planTree.getInputs().get(0);
71+
}
72+
6573
// first node must be RewriteAnchorNode
6674
rewrite(planTree, 0, planTree.getInputs().get(0));
75+
76+
// Validate after the entire rule group execution, not individual rules
77+
if (change > 0 && wholePlanBefore != null) {
78+
OptExpression wholePlanAfter = planTree.getInputs().get(0);
79+
PlanValidator.validateAfterRule(wholePlanBefore, wholePlanAfter, rules.get(0), context);
80+
}
81+
6782
// pushdownNotNullPredicates should task-bind, reset it before another RewriteTreeTask
6883
// TODO: refactor TaskContext to make it local to support this requirement better?
6984
context.getOptimizerContext().clearNotNullPredicates();
@@ -97,6 +112,7 @@ protected OptExpression applyRules(OptExpression parent, int childIndex, OptExpr
97112
}
98113

99114
OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, root);
115+
100116
List<OptExpression> result;
101117
try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.toString())) {
102118
result = rule.transform(root, context.getOptimizerContext());

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818
import com.starrocks.sql.optimizer.OptimizerContext;
1919
import com.starrocks.sql.optimizer.base.ColumnRefSet;
2020
import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
21+
import com.starrocks.sql.optimizer.validate.PlanValidator;
2122

2223
// The context for optimizer task
2324
public class TaskContext {
2425
private final OptimizerContext optimizerContext;
2526
private final PhysicalPropertySet requiredProperty;
2627
private ColumnRefSet requiredColumns;
2728
private double upperBoundCost;
29+
private final PlanValidator planValidator;
2830

2931
public TaskContext(OptimizerContext context,
3032
PhysicalPropertySet physicalPropertySet,
@@ -34,6 +36,11 @@ public TaskContext(OptimizerContext context,
3436
this.requiredProperty = physicalPropertySet;
3537
this.requiredColumns = requiredColumns;
3638
this.upperBoundCost = cost;
39+
this.planValidator = new PlanValidator();
40+
}
41+
42+
public PlanValidator getPlanValidator() {
43+
return planValidator;
3744
}
3845

3946
public OptimizerContext getOptimizerContext() {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/InputDependenciesChecker.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,15 +202,15 @@ private void checkInputCols(ColumnRefSet inputCols, ColumnRefSet usedCols, OptEx
202202
ColumnRefSet missedCols = usedCols.clone();
203203
missedCols.except(inputCols);
204204
if (!missedCols.isEmpty()) {
205-
String message = String.format("Invalid plan:%s%s%s The required cols %s cannot obtain from input cols %s.",
205+
String message = String.format("Invalid plan:%s%s\n%s \nThe required cols %s cannot obtain from input cols %s.",
206206
System.lineSeparator(), optExpr.debugString(), PREFIX, missedCols, inputCols);
207207
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
208208
}
209209
}
210210

211211
private void checkInputType(ColumnRefOperator inputCol, ColumnRefOperator outputCol, OptExpression optExpression) {
212212
if (!outputCol.getType().isFullyCompatible(inputCol.getType())) {
213-
String message = String.format("Invalid plan:%s%s%s Type of output col %s is not fully compatible with " +
213+
String message = String.format("Invalid plan:%s%s\n%s \nType of output col %s is not fully compatible with " +
214214
"type of input col %s.",
215215
System.lineSeparator(), optExpression.debugString(), PREFIX, outputCol, inputCol);
216216
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
@@ -219,7 +219,7 @@ private void checkInputType(ColumnRefOperator inputCol, ColumnRefOperator output
219219

220220
private void checkChildNumberOfSet(int inputSize, int requiredSize, OptExpression optExpression) {
221221
if (inputSize != requiredSize) {
222-
String message = String.format("Invalid plan:%s%s%s. The required number of children is %d but found %d.",
222+
String message = String.format("Invalid plan:%s%s\n%s. \nThe required number of children is %d but found %d.",
223223
System.lineSeparator(), optExpression.debugString(), PREFIX, requiredSize, inputSize);
224224
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
225225
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/PlanValidator.java‎

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.starrocks.sql.common.ErrorType;
2222
import com.starrocks.sql.common.StarRocksPlannerException;
2323
import com.starrocks.sql.optimizer.OptExpression;
24+
import com.starrocks.sql.optimizer.rule.Rule;
2425
import com.starrocks.sql.optimizer.task.TaskContext;
2526
import org.apache.logging.log4j.LogManager;
2627
import org.apache.logging.log4j.Logger;
@@ -31,22 +32,33 @@ public final class PlanValidator {
3132

3233
private static final Logger LOGGER = LogManager.getLogger(PlanValidator.class);
3334

34-
private static final PlanValidator INSTANCE = new PlanValidator();
35+
private List<Checker> checkerList;
36+
private boolean enableInputDependenciesChecker = false;
3537

36-
private final List<Checker> checkerList;
38+
public PlanValidator() {
39+
checkerList = ImmutableList.of(CTEUniqueChecker.getInstance());
40+
}
41+
42+
public void enableInputDependenciesChecker() {
43+
if (!enableInputDependenciesChecker) {
44+
checkerList = ImmutableList.of(
45+
CTEUniqueChecker.getInstance(),
46+
InputDependenciesChecker.getInstance());
47+
enableInputDependenciesChecker = true;
48+
}
49+
}
3750

38-
private PlanValidator() {
51+
/**
52+
* Called at final validation to enable all checkers including ColumnReuseChecker
53+
*/
54+
public void enableAllCheckers() {
3955
checkerList = ImmutableList.of(
40-
InputDependenciesChecker.getInstance(),
4156
TypeChecker.getInstance(),
4257
CTEUniqueChecker.getInstance(),
58+
InputDependenciesChecker.getInstance(),
4359
ColumnReuseChecker.getInstance());
4460
}
4561

46-
public static PlanValidator getInstance() {
47-
return INSTANCE;
48-
}
49-
5062
public void validatePlan(OptExpression optExpression, TaskContext taskContext) {
5163
boolean enablePlanValidation = ConnectContext.get().getSessionVariable().getEnablePlanValidation();
5264
try {
@@ -58,7 +70,7 @@ public void validatePlan(OptExpression optExpression, TaskContext taskContext) {
5870
} catch (IllegalArgumentException e) {
5971
String message = e.getMessage();
6072
if (!message.contains("Invalid plan")) {
61-
message = "Invalid plan:\n" + optExpression.debugString() + message;
73+
message = "Invalid plan:\n" + optExpression.debugString() + "\n" + message;
6274
}
6375
LOGGER.debug("Failed to validate plan.", e);
6476
if (enablePlanValidation) {
@@ -77,6 +89,44 @@ public void validatePlan(OptExpression optExpression, TaskContext taskContext) {
7789
}
7890
}
7991

92+
/**
93+
* Validates the query plan after applying an optimizer rule for debugging purposes.
94+
* Provides detailed error information including rule name and before/after expressions.
95+
*
96+
* @param beforeExpression The query plan expression before rule transformation
97+
* @param afterExpression The query plan expression after rule transformation
98+
* @param rule The optimizer rule that was applied
99+
* @param taskContext The task context containing session variables
100+
* @throws IllegalStateException if validation fails, with detailed rule information
101+
*/
102+
public static void validateAfterRule(OptExpression beforeExpression, OptExpression afterExpression,
103+
Rule rule, TaskContext taskContext) {
104+
if (!taskContext.getOptimizerContext().getSessionVariable().enableOptimizerRuleDebug()) {
105+
return;
106+
}
107+
108+
try {
109+
OptExpression clonedOpt = OptExpression.create(afterExpression.getOp(), afterExpression.getInputs());
110+
clonedOpt.clearStatsAndInitOutputInfo();
111+
taskContext.getPlanValidator().validatePlan(clonedOpt, taskContext);
112+
} catch (Exception e) {
113+
String errorMsg = String.format("Optimizer rule debug: Plan validation failed after applying rule [%s].\n" +
114+
"Validation error: %s\n" +
115+
"Hint: This error was caught by enable_optimizer_rule_debug=true\n" +
116+
"\n=== Query Plan Before Rule Application ===\n%s\n" +
117+
"\n=== Query Plan After Rule Application ===\n%s\n",
118+
rule.type(),
119+
e.getMessage(),
120+
beforeExpression.debugString(),
121+
afterExpression.debugString()
122+
);
123+
LOGGER.error(errorMsg, e);
124+
125+
throw new IllegalStateException(errorMsg, e);
126+
}
127+
}
128+
129+
80130
public interface Checker {
81131
void validate(OptExpression physicalPlan, TaskContext taskContext);
82132
}

0 commit comments

Comments
 (0)