Skip to content

Commit 26fc032

Browse files
stephen-shelbystephen-shelby
committed
[Enhancement] enable optimizer debug
Signed-off-by: stephen <[email protected]>
1 parent 7d81781 commit 26fc032

File tree

13 files changed

+325
-26
lines changed

13 files changed

+325
-26
lines changed

‎fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,8 @@ public static MaterializedViewRewriteMode parse(String str) {
782782

783783
public static final String ENABLE_PLAN_VALIDATION = "enable_plan_validation";
784784

785+
public static final String ENABLE_OPTIMIZER_RULE_DEBUG = "enable_optimizer_rule_debug";
786+
785787
public static final String ENABLE_STRICT_TYPE = "enable_strict_type";
786788

787789
public static final String PARTIAL_UPDATE_MODE = "partial_update_mode";
@@ -2463,6 +2465,9 @@ public long getConnectorSinkTargetMaxFileSize() {
24632465
@VarAttr(name = ENABLE_PLAN_VALIDATION, flag = VariableMgr.INVISIBLE)
24642466
private boolean enablePlanValidation = true;
24652467

2468+
@VarAttr(name = ENABLE_OPTIMIZER_RULE_DEBUG)
2469+
private boolean enableOptimizerRuleDebug = false;
2470+
24662471
@VarAttr(name = SCAN_OR_TO_UNION_LIMIT, flag = VariableMgr.INVISIBLE)
24672472
private int scanOrToUnionLimit = 4;
24682473

@@ -4731,6 +4736,14 @@ public void setEnablePlanValidation(boolean val) {
47314736
this.enablePlanValidation = val;
47324737
}
47334738

4739+
public boolean enableOptimizerRuleDebug() {
4740+
return this.enableOptimizerRuleDebug;
4741+
}
4742+
4743+
public void setEnableOptimizerRuleDebug(boolean enableOptimizerRuleDebug) {
4744+
this.enableOptimizerRuleDebug = enableOptimizerRuleDebug;
4745+
}
4746+
47344747
public boolean isCboPruneSubfield() {
47354748
return cboPruneSubfield;
47364749
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@
131131
import com.starrocks.sql.optimizer.task.TaskScheduler;
132132
import com.starrocks.sql.optimizer.validate.MVRewriteValidator;
133133
import com.starrocks.sql.optimizer.validate.OptExpressionValidator;
134-
import com.starrocks.sql.optimizer.validate.PlanValidator;
135134
import org.apache.commons.collections4.CollectionUtils;
136135
import org.apache.logging.log4j.LogManager;
137136
import org.apache.logging.log4j.Logger;
@@ -309,8 +308,9 @@ private OptExpression optimizeByCost(ConnectContext connectContext,
309308
mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add);
310309

311310
try (Timer ignored = Tracers.watchScope("PlanValidate")) {
311+
rootTaskContext.getPlanValidator().enableAllCheckers();
312312
// valid the final plan
313-
PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext);
313+
rootTaskContext.getPlanValidator().validatePlan(finalPlan, rootTaskContext);
314314
// validate mv and log tracer if needed
315315
MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators);
316316
mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext);
@@ -589,6 +589,9 @@ private OptExpression logicalRuleRewrite(
589589
rootTaskContext.setRequiredColumns(requiredColumns.clone());
590590
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
591591

592+
// Enable intermediate checkers after column pruning for rule transformation validation
593+
rootTaskContext.getPlanValidator().enableIntermediateCheckers();
594+
592595
pruneTables(tree, rootTaskContext, requiredColumns);
593596

594597
scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule());
@@ -663,7 +666,6 @@ private OptExpression logicalRuleRewrite(
663666

664667
scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE);
665668
scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
666-
667669
scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule());
668670
scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
669671
scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule());
@@ -699,10 +701,8 @@ private OptExpression logicalRuleRewrite(
699701
scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule());
700702
// before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction
701703
scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance());
702-
703704
// After this rule, we shouldn't generate logical project operator
704705
scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
705-
706706
scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule());
707707
scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule());
708708
// intersect rewrite depend on statistics
@@ -720,7 +720,6 @@ private OptExpression logicalRuleRewrite(
720720
scheduler.rewriteOnce(tree, rootTaskContext, JsonPathRewriteRule.createForOlapScan());
721721
scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMinMaxByMonotonicFunctionRule());
722722
scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT);
723-
724723
// NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle
725724
// select count(distinct c) from t group by a, b
726725
// if this rule has applied before MV.
@@ -975,7 +974,6 @@ void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTask
975974
private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
976975
OptExpression result) {
977976
Preconditions.checkState(result.getOp().isPhysical());
978-
979977
int planCount = result.getPlanCount();
980978

981979
// Since there may be many different plans in the logic phase, it's possible

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/RowOutputInfo.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ public List<ColumnOutputInfo> getCommonColInfo() {
137137
}
138138

139139
public List<ColumnRefOperator> getOutputColRefs() {
140-
return chooseOutputMap().values().stream().map(e -> e.getColumnRef()).collect(Collectors.toList());
140+
return chooseOutputMap().values().stream().map(ColumnOutputInfo::getColumnRef).collect(Collectors.toList());
141141
}
142142

143143
public Map<ColumnRefOperator, ScalarOperator> getColumnRefMap() {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/ColumnRefSet.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public ColumnRefSet(Collection<ColumnRefOperator> refs) {
4747

4848
public static ColumnRefSet createByIds(Collection<Integer> colIds) {
4949
ColumnRefSet columnRefSet = new ColumnRefSet();
50-
colIds.stream().forEach(columnRefSet::union);
50+
colIds.forEach(columnRefSet::union);
5151
return columnRefSet;
5252
}
5353

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggregateGroupingSetsRule.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ private OptExpression buildSelectConsume(ColumnRefFactory factory,
247247
/*
248248
* select a, b, c, d, null, sum(x) x from t group by rollup(a, b, c, d)
249249
*/
250-
private OptExpression buildSubRepeatConsume(ColumnRefFactory factory,
250+
public OptExpression buildSubRepeatConsume(ColumnRefFactory factory,
251251
Map<ColumnRefOperator, ColumnRefOperator> outputs,
252252
LogicalAggregationOperator aggregate, LogicalRepeatOperator repeat,
253253
int cteId) {
@@ -357,4 +357,4 @@ private OptExpression buildSubRepeatConsume(ColumnRefFactory factory,
357357
return OptExpression.create(projectOperator,
358358
OptExpression.create(newAggregate, OptExpression.create(newRepeat, OptExpression.create(consume))));
359359
}
360-
}
360+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public void execute() {
8989
}
9090
List<OptExpression> targetExpressions;
9191
OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, extractExpr);
92+
9293
try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.getClass().getSimpleName())) {
9394
targetExpressions = rule.transform(extractExpr, context.getOptimizerContext());
9495
} catch (StarRocksPlannerException e) {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.starrocks.sql.optimizer.operator.OperatorType;
2424
import com.starrocks.sql.optimizer.operator.pattern.Pattern;
2525
import com.starrocks.sql.optimizer.rule.Rule;
26+
import com.starrocks.sql.optimizer.validate.PlanValidator;
2627

2728
import java.util.List;
2829

@@ -62,8 +63,22 @@ public void execute() {
6263
.getOptimizerOptions().isRuleDisable(rule.type()))) {
6364
return;
6465
}
66+
67+
// Save whole tree state before rule group execution for validation
68+
OptExpression wholePlanBefore = null;
69+
if (context.getOptimizerContext().getSessionVariable().enableOptimizerRuleDebug()) {
70+
wholePlanBefore = planTree.getInputs().get(0);
71+
}
72+
6573
// first node must be RewriteAnchorNode
6674
rewrite(planTree, 0, planTree.getInputs().get(0));
75+
76+
// Validate after the entire rule group execution
77+
if (wholePlanBefore != null && change > 0) {
78+
OptExpression wholePlanAfter = planTree.getInputs().get(0);
79+
PlanValidator.validateAfterRule(wholePlanBefore, wholePlanAfter, rules.get(0), context);
80+
}
81+
6782
// pushdownNotNullPredicates should task-bind, reset it before another RewriteTreeTask
6883
// TODO: refactor TaskContext to make it local to support this requirement better?
6984
context.getOptimizerContext().clearNotNullPredicates();
@@ -97,6 +112,7 @@ protected OptExpression applyRules(OptExpression parent, int childIndex, OptExpr
97112
}
98113

99114
OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, root);
115+
100116
List<OptExpression> result;
101117
try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.toString())) {
102118
result = rule.transform(root, context.getOptimizerContext());

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818
import com.starrocks.sql.optimizer.OptimizerContext;
1919
import com.starrocks.sql.optimizer.base.ColumnRefSet;
2020
import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
21+
import com.starrocks.sql.optimizer.validate.PlanValidator;
2122

2223
// The context for optimizer task
2324
public class TaskContext {
2425
private final OptimizerContext optimizerContext;
2526
private final PhysicalPropertySet requiredProperty;
2627
private ColumnRefSet requiredColumns;
2728
private double upperBoundCost;
29+
private final PlanValidator planValidator;
2830

2931
public TaskContext(OptimizerContext context,
3032
PhysicalPropertySet physicalPropertySet,
@@ -34,6 +36,11 @@ public TaskContext(OptimizerContext context,
3436
this.requiredProperty = physicalPropertySet;
3537
this.requiredColumns = requiredColumns;
3638
this.upperBoundCost = cost;
39+
this.planValidator = new PlanValidator();
40+
}
41+
42+
public PlanValidator getPlanValidator() {
43+
return planValidator;
3744
}
3845

3946
public OptimizerContext getOptimizerContext() {

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/CTEUniqueChecker.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ public static CTEUniqueChecker getInstance() {
4343
return INSTANCE;
4444
}
4545

46-
4746
@Override
4847
public void validate(OptExpression physicalPlan, TaskContext taskContext) {
4948
CTEUniqueChecker.Visitor visitor = new CTEUniqueChecker.Visitor();

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/InputDependenciesChecker.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,15 +202,15 @@ private void checkInputCols(ColumnRefSet inputCols, ColumnRefSet usedCols, OptEx
202202
ColumnRefSet missedCols = usedCols.clone();
203203
missedCols.except(inputCols);
204204
if (!missedCols.isEmpty()) {
205-
String message = String.format("Invalid plan:%s%s%s The required cols %s cannot obtain from input cols %s.",
205+
String message = String.format("Invalid plan:%s%s\n%s \nThe required cols %s cannot obtain from input cols %s.",
206206
System.lineSeparator(), optExpr.debugString(), PREFIX, missedCols, inputCols);
207207
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
208208
}
209209
}
210210

211211
private void checkInputType(ColumnRefOperator inputCol, ColumnRefOperator outputCol, OptExpression optExpression) {
212212
if (!outputCol.getType().isFullyCompatible(inputCol.getType())) {
213-
String message = String.format("Invalid plan:%s%s%s Type of output col %s is not fully compatible with " +
213+
String message = String.format("Invalid plan:%s%s\n%s \nType of output col %s is not fully compatible with " +
214214
"type of input col %s.",
215215
System.lineSeparator(), optExpression.debugString(), PREFIX, outputCol, inputCol);
216216
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
@@ -219,7 +219,7 @@ private void checkInputType(ColumnRefOperator inputCol, ColumnRefOperator output
219219

220220
private void checkChildNumberOfSet(int inputSize, int requiredSize, OptExpression optExpression) {
221221
if (inputSize != requiredSize) {
222-
String message = String.format("Invalid plan:%s%s%s. The required number of children is %d but found %d.",
222+
String message = String.format("Invalid plan:%s%s\n%s. \nThe required number of children is %d but found %d.",
223223
System.lineSeparator(), optExpression.debugString(), PREFIX, requiredSize, inputSize);
224224
throw new StarRocksPlannerException(message, ErrorType.INTERNAL_ERROR);
225225
}

0 commit comments

Comments
 (0)