diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java index 4fd3e5e9af..9cc881336c 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java @@ -32,6 +32,8 @@ import org.apache.amoro.config.Configurations; import org.apache.amoro.config.shade.utils.ConfigShadeUtils; import org.apache.amoro.exception.AmoroRuntimeException; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.ProcessFactory; import org.apache.amoro.server.catalog.CatalogManager; import org.apache.amoro.server.catalog.DefaultCatalogManager; import org.apache.amoro.server.dashboard.DashboardServer; @@ -47,12 +49,15 @@ import org.apache.amoro.server.persistence.HttpSessionHandlerFactory; import org.apache.amoro.server.persistence.SqlSessionFactoryProvider; import org.apache.amoro.server.process.ProcessService; +import org.apache.amoro.server.process.TableProcessFactoryManager; +import org.apache.amoro.server.process.executor.ExecuteEngineManager; import org.apache.amoro.server.resource.ContainerMetadata; import org.apache.amoro.server.resource.Containers; import org.apache.amoro.server.resource.DefaultOptimizerManager; import org.apache.amoro.server.resource.OptimizerManager; import org.apache.amoro.server.scheduler.inline.InlineTableExecutors; import org.apache.amoro.server.table.DefaultTableManager; +import org.apache.amoro.server.table.DefaultTableRuntimeFactory; import org.apache.amoro.server.table.DefaultTableService; import org.apache.amoro.server.table.RuntimeHandlerChain; import org.apache.amoro.server.table.TableManager; @@ -61,6 +66,8 @@ import org.apache.amoro.server.terminal.TerminalManager; import org.apache.amoro.server.utils.ThriftServiceProxy; import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; +import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.shade.guava32.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; @@ -75,6 +82,7 @@ import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TTransportException; import org.apache.amoro.shade.thrift.org.apache.thrift.transport.TTransportFactory; import org.apache.amoro.shade.thrift.org.apache.thrift.transport.layered.TFramedTransport; +import org.apache.amoro.table.TableRuntimeFactory; import org.apache.amoro.utils.IcebergThreadPools; import org.apache.amoro.utils.JacksonUtil; import org.apache.commons.lang3.StringUtils; @@ -96,6 +104,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import java.util.stream.Collectors; public class AmoroServiceContainer { @@ -231,14 +240,32 @@ public void transitionToFollower() { public void startOptimizingService() throws Exception { TableRuntimeFactoryManager tableRuntimeFactoryManager = new TableRuntimeFactoryManager(); tableRuntimeFactoryManager.initialize(); + List tableRuntimeFactories = tableRuntimeFactoryManager.installedPlugins(); + Preconditions.checkArgument( + tableRuntimeFactories.size() == 1, "Only one table runtime factory is supported"); + TableRuntimeFactory tableRuntimeFactory = new DefaultTableRuntimeFactory(); + + TableProcessFactoryManager tableProcessFactoryManager = new TableProcessFactoryManager(); + tableProcessFactoryManager.initialize(); + List processFactories = tableProcessFactoryManager.installedPlugins(); + tableRuntimeFactory.initialize(processFactories); + + List actionCoordinators = + tableRuntimeFactoryManager.installedPlugins().stream() + .flatMap(f -> f.supportedCoordinators().stream()) + .collect(Collectors.toList()); + + ExecuteEngineManager executeEngineManager = new ExecuteEngineManager(); tableService = - new DefaultTableService(serviceConfig, catalogManager, tableRuntimeFactoryManager); + new DefaultTableService( + serviceConfig, catalogManager, Lists.newArrayList(tableRuntimeFactory)); optimizingService = new DefaultOptimizingService(serviceConfig, catalogManager, optimizerManager, tableService); - processService = new ProcessService(serviceConfig, tableService); + processService = + new ProcessService(serviceConfig, tableService, actionCoordinators, executeEngineManager); LOG.info("Setting up AMS table executors..."); InlineTableExecutors.getInstance().setup(tableService, serviceConfig); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java index f3b13e69a6..e44f6defa5 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java @@ -47,7 +47,7 @@ import org.apache.amoro.server.resource.OptimizerManager; import org.apache.amoro.server.resource.OptimizerThread; import org.apache.amoro.server.resource.QuotaProvider; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.RuntimeHandlerChain; import org.apache.amoro.server.table.TableService; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; @@ -142,16 +142,16 @@ public RuntimeHandlerChain getTableRuntimeHandler() { return tableHandlerChain; } - private void loadOptimizingQueues(List tableRuntimeList) { + private void loadOptimizingQueues(List tableRuntimeList) { List optimizerGroups = getAs(ResourceMapper.class, ResourceMapper::selectResourceGroups); List optimizers = getAs(OptimizerMapper.class, OptimizerMapper::selectAll); - Map> groupToTableRuntimes = + Map> groupToTableRuntimes = tableRuntimeList.stream().collect(Collectors.groupingBy(TableRuntime::getGroupName)); optimizerGroups.forEach( group -> { String groupName = group.getName(); - List tableRuntimes = groupToTableRuntimes.remove(groupName); + List tableRuntimes = groupToTableRuntimes.remove(groupName); OptimizingQueue optimizingQueue = new OptimizingQueue( catalogManager, @@ -275,7 +275,7 @@ public boolean cancelProcess(long processId) { return false; } long tableId = processMeta.getTableId(); - DefaultTableRuntime tableRuntime = (DefaultTableRuntime) tableService.getRuntime(tableId); + CompatibleTableRuntime tableRuntime = (CompatibleTableRuntime) tableService.getRuntime(tableId); if (tableRuntime == null) { return false; } @@ -368,7 +368,7 @@ private class TableRuntimeHandlerImpl extends RuntimeHandlerChain { @Override public void handleStatusChanged(TableRuntime tableRuntime, OptimizingStatus originalStatus) { - DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; + CompatibleTableRuntime defaultTableRuntime = (CompatibleTableRuntime) tableRuntime; if (!defaultTableRuntime.getOptimizingStatus().isProcessing()) { getOptionalQueueByGroup(defaultTableRuntime.getGroupName()) .ifPresent(q -> q.refreshTable(defaultTableRuntime)); @@ -377,7 +377,7 @@ public void handleStatusChanged(TableRuntime tableRuntime, OptimizingStatus orig @Override public void handleConfigChanged(TableRuntime runtime, TableConfiguration originalConfig) { - DefaultTableRuntime tableRuntime = (DefaultTableRuntime) runtime; + CompatibleTableRuntime tableRuntime = (CompatibleTableRuntime) runtime; String originalGroup = originalConfig.getOptimizingConfig().getOptimizerGroup(); if (!tableRuntime.getGroupName().equals(originalGroup)) { getOptionalQueueByGroup(originalGroup).ifPresent(q -> q.releaseTable(tableRuntime)); @@ -388,14 +388,14 @@ public void handleConfigChanged(TableRuntime runtime, TableConfiguration origina @Override public void handleTableAdded(AmoroTable table, TableRuntime runtime) { - DefaultTableRuntime tableRuntime = (DefaultTableRuntime) runtime; + CompatibleTableRuntime tableRuntime = (CompatibleTableRuntime) runtime; getOptionalQueueByGroup(tableRuntime.getGroupName()) .ifPresent(q -> q.refreshTable(tableRuntime)); } @Override public void handleTableRemoved(TableRuntime runtime) { - DefaultTableRuntime tableRuntime = (DefaultTableRuntime) runtime; + CompatibleTableRuntime tableRuntime = (CompatibleTableRuntime) runtime; getOptionalQueueByGroup(tableRuntime.getGroupName()) .ifPresent(queue -> queue.releaseTable(tableRuntime)); } @@ -405,8 +405,8 @@ protected void initHandler(List tableRuntimeList) { LOG.info("OptimizerManagementService begin initializing"); loadOptimizingQueues( tableRuntimeList.stream() - .filter(t -> t instanceof DefaultTableRuntime) - .map(t -> (DefaultTableRuntime) t) + .filter(t -> t instanceof CompatibleTableRuntime) + .map(t -> (CompatibleTableRuntime) t) .collect(Collectors.toList())); optimizerKeeper.start(); optimizingConfigWatcher.start(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java index 964d27c7b8..e8d2b8ce62 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java @@ -48,7 +48,7 @@ import org.apache.amoro.server.resource.OptimizerInstance; import org.apache.amoro.server.resource.OptimizerThread; import org.apache.amoro.server.resource.QuotaProvider; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.blocker.TableBlocker; import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; @@ -112,7 +112,7 @@ public OptimizingQueue( ResourceGroup optimizerGroup, QuotaProvider quotaProvider, Executor planExecutor, - List tableRuntimeList, + List tableRuntimeList, int maxPlanningParallelism) { Preconditions.checkNotNull(optimizerGroup, "Optimizer group can not be null"); this.planExecutor = planExecutor; @@ -128,7 +128,7 @@ public OptimizingQueue( tableRuntimeList.forEach(this::initTableRuntime); } - private void initTableRuntime(DefaultTableRuntime tableRuntime) { + private void initTableRuntime(CompatibleTableRuntime tableRuntime) { TableOptimizingProcess process = null; if (tableRuntime.getOptimizingStatus().isProcessing() && tableRuntime.getProcessId() != 0) { TableProcessMeta meta = @@ -172,7 +172,7 @@ public String getContainerName() { return optimizerGroup.getContainer(); } - public void refreshTable(DefaultTableRuntime tableRuntime) { + public void refreshTable(CompatibleTableRuntime tableRuntime) { if (tableRuntime.getOptimizingConfig().isEnabled() && !tableRuntime.getOptimizingStatus().isProcessing()) { LOG.info( @@ -185,7 +185,7 @@ public void refreshTable(DefaultTableRuntime tableRuntime) { } } - public void releaseTable(DefaultTableRuntime tableRuntime) { + public void releaseTable(CompatibleTableRuntime tableRuntime) { scheduler.removeTable(tableRuntime); List processList = tableQueue.stream() @@ -280,7 +280,7 @@ private void skipBlockedTables(Set skipTables) { } private void triggerAsyncPlanning( - DefaultTableRuntime tableRuntime, Set skipTables, long startTime) { + CompatibleTableRuntime tableRuntime, Set skipTables, long startTime) { LOG.info( "Trigger planning table {} by policy {}", tableRuntime.getTableIdentifier(), @@ -325,7 +325,7 @@ private void triggerAsyncPlanning( }); } - private TableOptimizingProcess planInternal(DefaultTableRuntime tableRuntime) { + private TableOptimizingProcess planInternal(CompatibleTableRuntime tableRuntime) { tableRuntime.beginPlanning(); try { ServerTableIdentifier identifier = tableRuntime.getTableIdentifier(); @@ -427,7 +427,7 @@ private class TableOptimizingProcess implements OptimizingProcess { private final Lock lock = new ReentrantLock(); private final long processId; private final OptimizingType optimizingType; - private final DefaultTableRuntime tableRuntime; + private final CompatibleTableRuntime tableRuntime; private final long planTime; private final long targetSnapshotId; private final long targetChangeSnapshotId; @@ -471,7 +471,7 @@ public TaskRuntime poll(OptimizerThread thread, boolean needQuotaChecking) { } public TableOptimizingProcess( - AbstractOptimizingPlanner planner, DefaultTableRuntime tableRuntime) { + AbstractOptimizingPlanner planner, CompatibleTableRuntime tableRuntime) { processId = planner.getProcessId(); this.tableRuntime = tableRuntime; optimizingType = planner.getOptimizingType(); @@ -485,7 +485,7 @@ public TableOptimizingProcess( } public TableOptimizingProcess( - DefaultTableRuntime tableRuntime, + CompatibleTableRuntime tableRuntime, TableProcessMeta processMeta, OptimizingProcessState processState) { this.tableRuntime = tableRuntime; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java index 0759f1e367..b9c8a9f16d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java @@ -22,7 +22,7 @@ import org.apache.amoro.resource.ResourceGroup; import org.apache.amoro.server.optimizing.sorter.QuotaOccupySorter; import org.apache.amoro.server.optimizing.sorter.SorterFactory; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.slf4j.Logger; @@ -45,7 +45,8 @@ public class SchedulingPolicy { private static final String SCHEDULING_POLICY_PROPERTY_NAME = "scheduling-policy"; - private final Map tableRuntimeMap = new HashMap<>(); + private final Map tableRuntimeMap = + new HashMap<>(); private volatile String policyName; private final Lock tableLock = new ReentrantLock(); private static final Map sorterFactoryCache = new ConcurrentHashMap<>(); @@ -84,7 +85,7 @@ public String name() { return policyName; } - public DefaultTableRuntime scheduleTable(Set skipSet) { + public CompatibleTableRuntime scheduleTable(Set skipSet) { tableLock.lock(); try { fillSkipSet(skipSet); @@ -97,7 +98,7 @@ public DefaultTableRuntime scheduleTable(Set skipSet) { } } - private Comparator createSorterByPolicy() { + private Comparator createSorterByPolicy() { if (sorterFactoryCache.get(policyName) != null) { SorterFactory sorterFactory = sorterFactoryCache.get(policyName); return sorterFactory.createComparator(); @@ -117,14 +118,14 @@ private void fillSkipSet(Set originalSet) { .forEach(tableRuntime -> originalSet.add(tableRuntime.getTableIdentifier())); } - private boolean isTablePending(DefaultTableRuntime tableRuntime) { + private boolean isTablePending(CompatibleTableRuntime tableRuntime) { return tableRuntime.getOptimizingStatus() == OptimizingStatus.PENDING && (tableRuntime.getLastOptimizedSnapshotId() != tableRuntime.getCurrentSnapshotId() || tableRuntime.getLastOptimizedChangeSnapshotId() != tableRuntime.getCurrentChangeSnapshotId()); } - public void addTable(DefaultTableRuntime tableRuntime) { + public void addTable(CompatibleTableRuntime tableRuntime) { tableLock.lock(); try { tableRuntimeMap.put(tableRuntime.getTableIdentifier(), tableRuntime); @@ -133,7 +134,7 @@ public void addTable(DefaultTableRuntime tableRuntime) { } } - public void removeTable(DefaultTableRuntime tableRuntime) { + public void removeTable(CompatibleTableRuntime tableRuntime) { tableLock.lock(); try { tableRuntimeMap.remove(tableRuntime.getTableIdentifier()); @@ -143,7 +144,7 @@ public void removeTable(DefaultTableRuntime tableRuntime) { } @VisibleForTesting - Map getTableRuntimeMap() { + Map getTableRuntimeMap() { return tableRuntimeMap; } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/DefaultTableMaintainerContext.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/DefaultTableMaintainerContext.java index ca47e22522..d2842296b3 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/DefaultTableMaintainerContext.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/DefaultTableMaintainerContext.java @@ -22,7 +22,7 @@ import org.apache.amoro.maintainer.MaintainerMetrics; import org.apache.amoro.maintainer.OptimizingInfo; import org.apache.amoro.maintainer.TableMaintainerContext; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableOrphanFilesCleaningMetrics; import org.apache.amoro.server.utils.HiveLocationUtil; import org.apache.amoro.table.MixedTable; @@ -31,20 +31,20 @@ import java.util.Set; /** - * Default implementation of TableMaintainerContext for AMS. Adapts DefaultTableRuntime to + * Default implementation of TableMaintainerContext for AMS. Adapts CompatibleTableRuntime to * TableMaintainerContext interface. */ public class DefaultTableMaintainerContext implements TableMaintainerContext { - private final DefaultTableRuntime tableRuntime; + private final CompatibleTableRuntime tableRuntime; private final MixedTable mixedTable; - public DefaultTableMaintainerContext(DefaultTableRuntime tableRuntime) { + public DefaultTableMaintainerContext(CompatibleTableRuntime tableRuntime) { this.tableRuntime = tableRuntime; this.mixedTable = null; } - public DefaultTableMaintainerContext(DefaultTableRuntime tableRuntime, MixedTable mixedTable) { + public DefaultTableMaintainerContext(CompatibleTableRuntime tableRuntime, MixedTable mixedTable) { this.tableRuntime = tableRuntime; this.mixedTable = mixedTable; } @@ -84,12 +84,12 @@ public Set getHiveLocationPaths() { return HiveLocationUtil.getHiveLocation(mixedTable); } - /** OptimizingInfo implementation based on DefaultTableRuntime. */ + /** OptimizingInfo implementation based on CompatibleTableRuntime. */ private static class DefaultOptimizingInfo implements OptimizingInfo { - private final DefaultTableRuntime tableRuntime; + private final CompatibleTableRuntime tableRuntime; - DefaultOptimizingInfo(DefaultTableRuntime tableRuntime) { + DefaultOptimizingInfo(CompatibleTableRuntime tableRuntime) { this.tableRuntime = tableRuntime; } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainerFactory.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainerFactory.java index ec1583fa1d..faa713b387 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainerFactory.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainerFactory.java @@ -24,7 +24,7 @@ import org.apache.amoro.formats.iceberg.maintainer.IcebergTableMaintainer; import org.apache.amoro.formats.iceberg.maintainer.MixedTableMaintainer; import org.apache.amoro.maintainer.TableMaintainer; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; import org.apache.amoro.table.MixedTable; import org.apache.iceberg.Table; @@ -40,7 +40,7 @@ public class TableMaintainerFactory { * @return IcebergTableMaintainer instance */ public static IcebergTableMaintainer createIcebergMaintainer( - Table table, DefaultTableRuntime tableRuntime) { + Table table, CompatibleTableRuntime tableRuntime) { return new IcebergTableMaintainer( table, tableRuntime.getTableIdentifier().getIdentifier(), @@ -55,8 +55,8 @@ public static IcebergTableMaintainer createIcebergMaintainer( * @return TableMaintainer instance */ public static TableMaintainer create(AmoroTable amoroTable, TableRuntime tableRuntime) { - Preconditions.checkArgument(tableRuntime instanceof DefaultTableRuntime); - DefaultTableRuntime runtime = (DefaultTableRuntime) tableRuntime; + Preconditions.checkArgument(tableRuntime instanceof CompatibleTableRuntime); + CompatibleTableRuntime runtime = (CompatibleTableRuntime) tableRuntime; TableFormat format = amoroTable.format(); if (format.in(TableFormat.MIXED_HIVE, TableFormat.MIXED_ICEBERG)) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainers.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainers.java index eb152bb096..ebff392d26 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainers.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/maintainer/TableMaintainers.java @@ -23,7 +23,7 @@ import org.apache.amoro.TableRuntime; import org.apache.amoro.formats.iceberg.maintainer.MixedTableMaintainer; import org.apache.amoro.maintainer.TableMaintainer; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.table.MixedTable; import org.apache.iceberg.Table; @@ -42,12 +42,13 @@ public static TableMaintainer create(AmoroTable amoroTable, TableRuntime tabl } /** - * Create a {@link TableMaintainer} for the given table with DefaultTableRuntime. + * Create a {@link TableMaintainer} for the given table with CompatibleTableRuntime. * * @deprecated since 0.9.0, will be removed in 0.10.0. Use {@link - * TableMaintainerFactory#createIcebergMaintainer(Table, DefaultTableRuntime)} instead. + * TableMaintainerFactory#createIcebergMaintainer(Table, CompatibleTableRuntime)} instead. */ - public static TableMaintainer create(AmoroTable amoroTable, DefaultTableRuntime tableRuntime) { + public static TableMaintainer create( + AmoroTable amoroTable, CompatibleTableRuntime tableRuntime) { TableFormat format = amoroTable.format(); if (format.in(TableFormat.MIXED_HIVE, TableFormat.MIXED_ICEBERG)) { MixedTable mixedTable = (MixedTable) amoroTable.originalTable(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/BalancedSorter.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/BalancedSorter.java index d3b08e8773..d6e55b7327 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/BalancedSorter.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/BalancedSorter.java @@ -18,7 +18,7 @@ package org.apache.amoro.server.optimizing.sorter; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import java.util.Comparator; @@ -32,10 +32,10 @@ public String getIdentifier() { } @Override - public Comparator createComparator() { - return new Comparator() { + public Comparator createComparator() { + return new Comparator() { @Override - public int compare(DefaultTableRuntime one, DefaultTableRuntime another) { + public int compare(CompatibleTableRuntime one, CompatibleTableRuntime another) { return Long.compare( Math.max( one.getLastFullOptimizingTime(), diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/QuotaOccupySorter.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/QuotaOccupySorter.java index 3b448b2c25..7bccb789fa 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/QuotaOccupySorter.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/sorter/QuotaOccupySorter.java @@ -18,7 +18,7 @@ package org.apache.amoro.server.optimizing.sorter; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import java.util.Comparator; @@ -34,15 +34,15 @@ public String getIdentifier() { } @Override - public Comparator createComparator() { - return new Comparator() { - final Map tableWeightMap = Maps.newHashMap(); + public Comparator createComparator() { + return new Comparator() { + final Map tableWeightMap = Maps.newHashMap(); @Override - public int compare(DefaultTableRuntime one, DefaultTableRuntime another) { + public int compare(CompatibleTableRuntime one, CompatibleTableRuntime another) { return Double.compare( - tableWeightMap.computeIfAbsent(one, DefaultTableRuntime::calculateQuotaOccupy), - tableWeightMap.computeIfAbsent(another, DefaultTableRuntime::calculateQuotaOccupy)); + tableWeightMap.computeIfAbsent(one, CompatibleTableRuntime::calculateQuotaOccupy), + tableWeightMap.computeIfAbsent(another, CompatibleTableRuntime::calculateQuotaOccupy)); } }; } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/mapper/ProcessStateMapper.java b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/mapper/ProcessStateMapper.java deleted file mode 100644 index f7b7557819..0000000000 --- a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/mapper/ProcessStateMapper.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.server.persistence.mapper; - -import org.apache.amoro.process.TableProcessState; -import org.apache.ibatis.annotations.Insert; -import org.apache.ibatis.annotations.Options; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Result; -import org.apache.ibatis.annotations.ResultMap; -import org.apache.ibatis.annotations.Results; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; - -import java.util.Map; - -public interface ProcessStateMapper { - - @Insert( - "INSERT INTO table_process_state " - + "(process_id, action, table_id, retry_num, status, start_time, end_time, fail_reason, summary) " - + "VALUES " - + "(#{id}, #{action}, #{tableId}, #{retryNumber}, #{status}, #{startTime}, #{endTime}, #{failedReason}, #{summary})") - @Options(useGeneratedKeys = true, keyProperty = "id") - void createProcessState(TableProcessState state); - - @Update( - "UPDATE table_process_state " - + "SET status = #{status}, start_time = #{startTime} " - + "WHERE process_id = #{id} and retry_num = #{retryNumber}") - void updateProcessRunning(TableProcessState state); - - @Update( - "UPDATE table_process_state " - + "SET status = #{status}, end_time = #{endTime} " - + "WHERE process_id = #{id} and retry_num = #{retryNumber}") - void updateProcessCompleted(TableProcessState state); - - @Update( - "UPDATE table_process_state " - + "SET status = #{status}, end_time = #{endTime}, fail_reason = #{failedReason} " - + "WHERE process_id = #{id} and retry_num = #{retryNumber}") - void updateProcessFailed(TableProcessState state); - - @Select( - "SELECT process_id, action, table_id, retry_num, status, start_time, end_time, fail_reason, summary " - + "FROM table_process_state " - + "WHERE process_id = #{processId}") - @Results( - id = "TableProcessStateResultMap", - value = { - @Result(property = "id", column = "process_id"), - @Result(property = "action", column = "action"), - @Result(property = "tableId", column = "table_id"), - @Result(property = "retryNumber", column = "retry_num"), - @Result(property = "status", column = "status"), - @Result(property = "startTime", column = "start_time"), - @Result(property = "endTime", column = "end_time"), - @Result(property = "failedReason", column = "fail_reason"), - @Result(property = "summary", column = "summary", javaType = Map.class) - }) - TableProcessState getProcessStateById(@Param("processId") long processId); - - /** Query TableProcessState by table_id */ - @Select( - "SELECT process_id, action, table_id, retry_num, status, start_time, end_time, fail_reason, summary " - + "FROM table_process_state " - + "WHERE table_id = #{tableId}") - @ResultMap("TableProcessStateResultMap") - TableProcessState getProcessStateByTableId(@Param("tableId") long tableId); -} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinatorScheduler.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinatorScheduler.java index 83574c6636..daf5869413 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinatorScheduler.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinatorScheduler.java @@ -20,6 +20,7 @@ import org.apache.amoro.TableFormat; import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.ActionCoordinator; import org.apache.amoro.process.TableProcess; import org.apache.amoro.process.TableProcessStore; import org.apache.amoro.server.scheduler.PeriodicTableScheduler; @@ -27,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Optional; + /** * Periodic scheduler that delegates scheduling decisions to an {@link ActionCoordinator}. It * creates, recovers and retries table processes via {@link ProcessService}. @@ -95,8 +98,8 @@ protected boolean enabled(TableRuntime tableRuntime) { */ @Override protected void execute(TableRuntime tableRuntime) { - TableProcess process = coordinator.createTableProcess(tableRuntime); - processService.register(tableRuntime, process); + Optional process = coordinator.trigger(tableRuntime); + process.ifPresent(p -> processService.register(tableRuntime, p)); } /** @@ -110,16 +113,6 @@ protected void recover(TableRuntime tableRuntime, TableProcessStore processStore processService.recover(tableRuntime, process); } - /** - * Retry a failed table process. - * - * @param process process to retry - */ - protected void retry(TableProcess process) { - process = coordinator.retryTableProcess(process); - processService.retry(process); - } - /** * Get executor delay from coordinator. * diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java index ee9f136dc6..78a5ac7c77 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java @@ -25,15 +25,16 @@ import org.apache.amoro.TableRuntime; import org.apache.amoro.config.Configurations; import org.apache.amoro.config.TableConfiguration; +import org.apache.amoro.process.ActionCoordinator; import org.apache.amoro.process.ProcessEvent; import org.apache.amoro.process.ProcessStatus; import org.apache.amoro.process.TableProcess; -import org.apache.amoro.server.manager.AbstractPluginManager; import org.apache.amoro.server.optimizing.OptimizingStatus; import org.apache.amoro.server.persistence.PersistentBase; import org.apache.amoro.server.persistence.mapper.TableProcessMapper; import org.apache.amoro.server.process.executor.EngineType; import org.apache.amoro.server.process.executor.ExecuteEngine; +import org.apache.amoro.server.process.executor.ExecuteEngineManager; import org.apache.amoro.server.process.executor.TableProcessExecutor; import org.apache.amoro.server.table.RuntimeHandlerChain; import org.apache.amoro.server.table.TableService; @@ -59,11 +60,11 @@ public class ProcessService extends PersistentBase { private static final Logger LOG = LoggerFactory.getLogger(ProcessService.class); private final TableService tableService; + private final List actionCoordinatorLists; private final Map actionCoordinators = new ConcurrentHashMap<>(); private final Map executeEngines = new ConcurrentHashMap<>(); - private final ActionCoordinatorManager actionCoordinatorManager; private final ExecuteEngineManager executeEngineManager; private final ProcessRuntimeHandler tableRuntimeHandler = new ProcessRuntimeHandler(); private final ThreadPoolExecutor processExecutionPool = @@ -72,17 +73,13 @@ public class ProcessService extends PersistentBase { private final Map> activeTableProcess = new ConcurrentHashMap<>(); - public ProcessService(Configurations serviceConfig, TableService tableService) { - this(serviceConfig, tableService, new ActionCoordinatorManager(), new ExecuteEngineManager()); - } - public ProcessService( Configurations serviceConfig, TableService tableService, - ActionCoordinatorManager actionCoordinatorManager, + List actionCoordinators, ExecuteEngineManager executeEngineManager) { this.tableService = tableService; - this.actionCoordinatorManager = actionCoordinatorManager; + this.actionCoordinatorLists = actionCoordinators; this.executeEngineManager = executeEngineManager; } @@ -126,15 +123,6 @@ public void recover(TableRuntime tableRuntime, TableProcess process) { executeOrTraceProcess(process); } - /** - * Retry a failed table process. - * - * @param process process to retry - */ - public void retry(TableProcess process) { - executeOrTraceProcess(process); - } - /** * Cancel a table process and release related resources. * @@ -147,7 +135,7 @@ public void cancel(TableProcess process) { /** Dispose the service, shutdown engines and clear active processes. */ public void dispose() { - actionCoordinatorManager.close(); + this.actionCoordinators.values().forEach(RuntimeHandlerChain::dispose); executeEngineManager.close(); processExecutionPool.shutdown(); activeTableProcess.clear(); @@ -155,16 +143,12 @@ public void dispose() { private void initialize(List tableRuntimes) { LOG.info("Initializing process service"); - actionCoordinatorManager.initialize(); - actionCoordinatorManager - .installedPlugins() - .forEach( - actionCoordinator -> { - actionCoordinators.put( - actionCoordinator.action().getName(), - new ActionCoordinatorScheduler( - actionCoordinator, tableService, ProcessService.this)); - }); + actionCoordinatorLists.forEach( + actionCoordinator -> { + actionCoordinators.put( + actionCoordinator.action().getName(), + new ActionCoordinatorScheduler(actionCoordinator, tableService, ProcessService.this)); + }); executeEngineManager.initialize(); executeEngineManager .installedPlugins() @@ -242,7 +226,7 @@ private void executeOrTraceProcess(TableProcess process) { "Regular Retry.", process.getProcessParameters(), process.getSummary()); - scheduler.retry(process); + executeOrTraceProcess(process); } else { untrackTableProcessInstance( process.getTableRuntime().getTableIdentifier(), process.getId()); @@ -551,18 +535,4 @@ protected void doDispose() { // TODO: dispose } } - - /** Manager for {@link ActionCoordinator} plugins. */ - public static class ActionCoordinatorManager extends AbstractPluginManager { - public ActionCoordinatorManager() { - super("action-coordinators"); - } - } - - /** Manager for {@link ExecuteEngine} plugins. */ - public static class ExecuteEngineManager extends AbstractPluginManager { - public ExecuteEngineManager() { - super("execute-engines"); - } - } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessFactoryManager.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessFactoryManager.java new file mode 100644 index 0000000000..c04bde9e3b --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessFactoryManager.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.process; + +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.server.manager.AbstractPluginManager; + +public class TableProcessFactoryManager extends AbstractPluginManager { + public TableProcessFactoryManager() { + super("process-factories"); + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessMeta.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessMeta.java index 311b60e035..6890c25c10 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessMeta.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/TableProcessMeta.java @@ -19,7 +19,6 @@ package org.apache.amoro.server.process; import org.apache.amoro.process.ProcessStatus; -import org.apache.amoro.process.TableProcessState; import org.apache.amoro.process.TableProcessStore; import java.util.HashMap; @@ -189,25 +188,6 @@ public static TableProcessMeta fromTableProcessStore(TableProcessStore tableProc return tableProcessMeta; } - @Deprecated - public static TableProcessMeta fromTableProcessState(TableProcessState tableProcessState) { - TableProcessMeta tableProcessMeta = new TableProcessMeta(); - tableProcessMeta.setProcessId(tableProcessState.getId()); - tableProcessMeta.setTableId(tableProcessState.getTableIdentifier().getId()); - tableProcessMeta.setExternalProcessIdentifier(tableProcessState.getExternalProcessIdentifier()); - tableProcessMeta.setStatus(tableProcessState.getStatus()); - tableProcessMeta.setProcessType(tableProcessState.getAction().getName()); - tableProcessMeta.setProcessStage(tableProcessState.getStage().getDesc()); - tableProcessMeta.setExecutionEngine(tableProcessState.getExecutionEngine()); - tableProcessMeta.setRetryNumber(tableProcessState.getRetryNumber()); - tableProcessMeta.setCreateTime(tableProcessState.getStartTime()); - tableProcessMeta.setFinishTime(tableProcessState.getEndTime()); - tableProcessMeta.setFailMessage(tableProcessState.getFailedReason()); - tableProcessMeta.setProcessParameters(tableProcessState.getProcessParameters()); - tableProcessMeta.setSummary(tableProcessState.getSummary()); - return tableProcessMeta; - } - public static TableProcessMeta of( long processId, long tableId, diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/ExecuteEngineManager.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/ExecuteEngineManager.java new file mode 100644 index 0000000000..77921b882a --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/executor/ExecuteEngineManager.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.process.executor; + +import org.apache.amoro.server.manager.AbstractPluginManager; + +/** Manager for {@link org.apache.amoro.server.process.executor.ExecuteEngine} plugins. */ +public class ExecuteEngineManager extends AbstractPluginManager { + public ExecuteEngineManager() { + super("execute-engines"); + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicExternalScheduler.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicExternalScheduler.java deleted file mode 100644 index 346f02772b..0000000000 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicExternalScheduler.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.server.scheduler; - -import org.apache.amoro.Action; -import org.apache.amoro.SupportsProcessPlugins; -import org.apache.amoro.TableRuntime; -import org.apache.amoro.process.AmoroProcess; -import org.apache.amoro.process.ManagedProcess; -import org.apache.amoro.process.ProcessFactory; -import org.apache.amoro.process.ProcessStatus; -import org.apache.amoro.process.SimpleFuture; -import org.apache.amoro.process.TableProcess; -import org.apache.amoro.process.TableProcessState; -import org.apache.amoro.process.TableProcessStore; -import org.apache.amoro.resource.ExternalResourceContainer; -import org.apache.amoro.resource.Resource; -import org.apache.amoro.resource.ResourceManager; -import org.apache.amoro.server.persistence.PersistentBase; -import org.apache.amoro.server.persistence.mapper.TableProcessMapper; -import org.apache.amoro.server.process.DefaultTableProcessStore; -import org.apache.amoro.server.process.TableProcessMeta; -import org.apache.amoro.server.table.TableService; -import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; - -import java.util.List; -import java.util.Optional; - -public abstract class PeriodicExternalScheduler extends PeriodicTableScheduler { - - private final ExternalResourceContainer resourceContainer; - private final ResourceManager resourceManager; - private final ProcessFactory processFactory; - - public PeriodicExternalScheduler( - ResourceManager resourceManager, - ExternalResourceContainer resourceContainer, - Action action, - TableService tableService, - int poolSize) { - super(action, tableService, poolSize); - this.resourceContainer = resourceContainer; - this.resourceManager = resourceManager; - this.processFactory = generateProcessFactory(); - } - - @Override - protected void initHandler(List tableRuntimeList) { - tableRuntimeList.stream() - .filter(t -> t instanceof SupportsProcessPlugins) - .map(t -> (SupportsProcessPlugins) t) - .forEach(tableRuntime -> tableRuntime.install(getAction(), processFactory)); - super.initHandler(tableRuntimeList); - } - - @Override - protected boolean enabled(TableRuntime tableRuntime) { - return Optional.of(tableRuntime) - .filter(t -> t instanceof SupportsProcessPlugins) - .map(t -> (SupportsProcessPlugins) t) - .map(t -> t.enabled(getAction())) - .orElse(false); - } - - @Override - protected void execute(TableRuntime tableRuntime) { - Preconditions.checkArgument(tableRuntime instanceof SupportsProcessPlugins); - SupportsProcessPlugins runtimeSupportProcessPlugin = (SupportsProcessPlugins) tableRuntime; - // Trigger a table process and check conflicts by table runtime - // Update process state after process completed, the callback must be register first - AmoroProcess process = runtimeSupportProcessPlugin.trigger(getAction()); - process.getCompleteFuture().whenCompleted(() -> persistTableProcess(process)); - ManagedProcess managedProcess = new ManagedTableProcess(process); - - // Submit the table process to resource manager, this is a sync operation - // update process completed and delete related resources - managedProcess.submit(); - - // Trace the table process by async framework so that process can be called back when completed - trace(process); - } - - protected int getMaxRetryNumber() { - return 1; - } - - protected abstract void trace(AmoroProcess process); - - protected ProcessFactory generateProcessFactory() { - return new ExternalProcessFactory(); - } - - protected void persistTableProcess(AmoroProcess process) { - TableProcessStore store = process.store(); - if (store.getStatus() == ProcessStatus.SUBMITTED) { - new PersistencyHelper().createProcessState(store); - } else { - new PersistencyHelper().updateProcessStatus(store); - } - } - - private static class PersistencyHelper extends PersistentBase { - - void createProcessState(TableProcessStore store) { - TableProcessMeta meta = TableProcessMeta.fromTableProcessStore(store); - doAs( - TableProcessMapper.class, - mapper -> - mapper.updateProcess( - meta.getTableId(), - meta.getProcessId(), - meta.getExternalProcessIdentifier(), - meta.getStatus(), - meta.getProcessStage(), - meta.getRetryNumber(), - meta.getFinishTime(), - meta.getFailMessage(), - meta.getProcessParameters(), - meta.getSummary())); - } - - void updateProcessStatus(TableProcessStore store) { - TableProcessMeta meta = TableProcessMeta.fromTableProcessStore(store); - doAs( - TableProcessMapper.class, - mapper -> - mapper.updateProcess( - meta.getTableId(), - meta.getProcessId(), - meta.getExternalProcessIdentifier(), - meta.getStatus(), - meta.getProcessStage(), - meta.getRetryNumber(), - meta.getFinishTime(), - meta.getFailMessage(), - meta.getProcessParameters(), - meta.getSummary())); - } - } - - private class ExternalTableProcess extends TableProcess { - - ExternalTableProcess(TableRuntime tableRuntime) { - super( - tableRuntime, - new DefaultTableProcessStore( - tableRuntime, new TableProcessMeta(), PeriodicExternalScheduler.this.getAction())); - } - - ExternalTableProcess(TableRuntime tableRuntime, TableProcessState state) { - super( - tableRuntime, - new DefaultTableProcessStore( - tableRuntime, - TableProcessMeta.fromTableProcessState(state), - PeriodicExternalScheduler.this.getAction())); - } - - @Override - protected void closeInternal() {} - } - - private class ExternalProcessFactory implements ProcessFactory { - - @Override - public AmoroProcess create(TableRuntime tableRuntime, Action action) { - return new ExternalTableProcess(tableRuntime); - } - - @Override - public AmoroProcess recover(TableRuntime tableRuntime, TableProcessState state) { - return new ExternalTableProcess(tableRuntime, state); - } - } - - protected class ManagedTableProcess implements ManagedProcess { - - private final AmoroProcess process; - - ManagedTableProcess(AmoroProcess process) { - this.process = process; - } - - @Override - public void submit() { - Resource resource = resourceContainer.submit(this); - if (resource == null) { - throw new IllegalStateException("Submit table process can not return null resource"); - } - persistTableProcess(this); - resourceManager.createResource(resource); - getCompleteFuture() - .whenCompleted( - () -> { - resourceManager.deleteResource(resource.getResourceId()); - if (store().getStatus() == ProcessStatus.FAILED - && store().getRetryNumber() < getMaxRetryNumber()) { - retry(); - } - }); - store().begin().updateTableProcessStatus(ProcessStatus.SUBMITTED).commit(); - getSubmitFuture().complete(); - } - - @Override - public void complete() { - store() - .begin() - .updateTableProcessStatus(ProcessStatus.SUCCESS) - .updateFinishTime(System.currentTimeMillis()) - .commit(); - process.getCompleteFuture().complete(); - } - - @Override - public void complete(String failedReason) { - store() - .begin() - .updateTableProcessStatus(ProcessStatus.FAILED) - .updateTableProcessFailMessage(failedReason) - .updateFinishTime(System.currentTimeMillis()) - .commit(); - process.getCompleteFuture().complete(); - } - - @Override - public void retry() { - store() - .begin() - .updateTableProcessStatus(ProcessStatus.PENDING) - .updateRetryNumber(store().getRetryNumber() + 1) - .updateExternalProcessIdentifier("") - .commit(); - submit(); - } - - @Override - public void kill() { - store() - .begin() - .updateTableProcessStatus(ProcessStatus.KILLED) - .updateFinishTime(System.currentTimeMillis()) - .commit(); - process.getCompleteFuture().complete(); - } - - @Override - public SimpleFuture getSubmitFuture() { - return process.getSubmitFuture(); - } - - @Override - public SimpleFuture getCompleteFuture() { - return process.getCompleteFuture(); - } - - @Override - public TableProcessStore store() { - return process.store(); - } - } -} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicTableScheduler.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicTableScheduler.java index ef74193258..5ecc6dcfd3 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicTableScheduler.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/PeriodicTableScheduler.java @@ -25,7 +25,7 @@ import org.apache.amoro.TableRuntime; import org.apache.amoro.config.TableConfiguration; import org.apache.amoro.server.optimizing.OptimizingStatus; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.RuntimeHandlerChain; import org.apache.amoro.server.table.TableService; import org.apache.amoro.server.table.cleanup.CleanupOperation; @@ -162,7 +162,7 @@ private void persistUpdatingCleanupTime(TableRuntime tableRuntime) { try { long currentTime = System.currentTimeMillis(); - ((DefaultTableRuntime) tableRuntime).updateLastCleanTime(cleanupOperation, currentTime); + ((CompatibleTableRuntime) tableRuntime).updateLastCleanTime(cleanupOperation, currentTime); logger.debug( "Update lastCleanTime for table {} with cleanup operation {}", @@ -193,7 +193,7 @@ protected boolean shouldExecuteTask( } long lastCleanupEndTime = - ((DefaultTableRuntime) tableRuntime).getLastCleanTime(cleanupOperation); + ((CompatibleTableRuntime) tableRuntime).getLastCleanTime(cleanupOperation); // If it's zero, execute the task if (lastCleanupEndTime == 0L) { @@ -233,9 +233,9 @@ private boolean shouldSkipOperation( return true; } - if (!(tableRuntime instanceof DefaultTableRuntime)) { + if (!(tableRuntime instanceof CompatibleTableRuntime)) { logger.debug( - "Table runtime is not DefaultTableRuntime, skipping cleanup time check for table {}", + "Table runtime is not CompatibleTableRuntime, skipping cleanup time check for table {}", tableRuntime.getTableIdentifier().getTableName()); return true; } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java index 3ad669092f..ff5b484b57 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/DanglingDeleteFilesCleaningExecutor.java @@ -24,7 +24,7 @@ import org.apache.amoro.maintainer.TableMaintainer; import org.apache.amoro.server.optimizing.maintainer.TableMaintainers; import org.apache.amoro.server.scheduler.PeriodicTableScheduler; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableService; import org.apache.amoro.server.table.cleanup.CleanupOperation; import org.slf4j.Logger; @@ -64,7 +64,7 @@ protected CleanupOperation getCleanupOperation() { @Override protected boolean enabled(TableRuntime tableRuntime) { - return tableRuntime instanceof DefaultTableRuntime + return tableRuntime instanceof CompatibleTableRuntime && tableRuntime.getTableConfiguration().isDeleteDanglingDeleteFilesEnabled(); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/OptimizingCommitExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/OptimizingCommitExecutor.java index 8ff5da03a1..5f04cd2c11 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/OptimizingCommitExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/OptimizingCommitExecutor.java @@ -21,7 +21,7 @@ import org.apache.amoro.TableRuntime; import org.apache.amoro.server.optimizing.OptimizingStatus; import org.apache.amoro.server.scheduler.PeriodicTableScheduler; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableService; import java.util.Optional; @@ -42,8 +42,8 @@ protected long getNextExecutingTime(TableRuntime tableRuntime) { @Override protected boolean enabled(TableRuntime tableRuntime) { return Optional.of(tableRuntime) - .filter(t -> t instanceof DefaultTableRuntime) - .map(t -> (DefaultTableRuntime) t) + .filter(t -> t instanceof CompatibleTableRuntime) + .map(t -> (CompatibleTableRuntime) t) .map(t -> t.getOptimizingStatus() == OptimizingStatus.COMMITTING) .orElse(false); } @@ -56,9 +56,9 @@ protected long getExecutorDelay() { @Override protected void execute(TableRuntime tableRuntime) { Optional.of(tableRuntime) - .filter(t -> t instanceof DefaultTableRuntime) - .map(t -> (DefaultTableRuntime) t) - .map(DefaultTableRuntime::getOptimizingProcess) + .filter(t -> t instanceof CompatibleTableRuntime) + .map(t -> (CompatibleTableRuntime) t) + .map(CompatibleTableRuntime::getOptimizingProcess) .orElseThrow( () -> new IllegalStateException( diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java index 18071a2cdd..796e97ef2a 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java @@ -28,7 +28,7 @@ import org.apache.amoro.server.optimizing.OptimizingProcess; import org.apache.amoro.server.optimizing.OptimizingStatus; import org.apache.amoro.server.scheduler.PeriodicTableScheduler; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableService; import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; @@ -50,17 +50,17 @@ public TableRuntimeRefreshExecutor( @Override protected boolean enabled(TableRuntime tableRuntime) { - return tableRuntime instanceof DefaultTableRuntime; + return tableRuntime instanceof CompatibleTableRuntime; } @Override protected long getNextExecutingTime(TableRuntime tableRuntime) { - DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; + CompatibleTableRuntime defaultTableRuntime = (CompatibleTableRuntime) tableRuntime; return Math.min( defaultTableRuntime.getOptimizingConfig().getMinorLeastInterval() * 4L / 5, interval); } - private void tryEvaluatingPendingInput(DefaultTableRuntime tableRuntime, MixedTable table) { + private void tryEvaluatingPendingInput(CompatibleTableRuntime tableRuntime, MixedTable table) { // only evaluate pending input when optimizing is enabled and in idle state OptimizingConfig optimizingConfig = tableRuntime.getOptimizingConfig(); if (optimizingConfig.isEnabled() @@ -94,8 +94,8 @@ private void tryEvaluatingPendingInput(DefaultTableRuntime tableRuntime, MixedTa @Override public void handleConfigChanged(TableRuntime tableRuntime, TableConfiguration originalConfig) { - Preconditions.checkArgument(tableRuntime instanceof DefaultTableRuntime); - DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; + Preconditions.checkArgument(tableRuntime instanceof CompatibleTableRuntime); + CompatibleTableRuntime defaultTableRuntime = (CompatibleTableRuntime) tableRuntime; // After disabling self-optimizing, close the currently running optimizing process. if (originalConfig.getOptimizingConfig().isEnabled() && !tableRuntime.getTableConfiguration().getOptimizingConfig().isEnabled()) { @@ -114,8 +114,8 @@ protected long getExecutorDelay() { @Override public void execute(TableRuntime tableRuntime) { try { - Preconditions.checkArgument(tableRuntime instanceof DefaultTableRuntime); - DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; + Preconditions.checkArgument(tableRuntime instanceof CompatibleTableRuntime); + CompatibleTableRuntime defaultTableRuntime = (CompatibleTableRuntime) tableRuntime; long lastOptimizedSnapshotId = defaultTableRuntime.getLastOptimizedSnapshotId(); long lastOptimizedChangeSnapshotId = defaultTableRuntime.getLastOptimizedChangeSnapshotId(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/AbstractTableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/AbstractTableRuntime.java index e07cd4432d..68b4e1bd4c 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/AbstractTableRuntime.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/AbstractTableRuntime.java @@ -18,17 +18,26 @@ package org.apache.amoro.server.table; +import org.apache.amoro.Action; import org.apache.amoro.ServerTableIdentifier; -import org.apache.amoro.SupportsProcessPlugins; import org.apache.amoro.TableRuntime; import org.apache.amoro.config.TableConfiguration; +import org.apache.amoro.process.TableProcessStore; import org.apache.amoro.server.persistence.PersistentBase; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.shade.zookeeper3.org.apache.curator.shaded.com.google.common.collect.Maps; import org.apache.amoro.table.TableRuntimeStore; -public abstract class AbstractTableRuntime extends PersistentBase - implements TableRuntime, SupportsProcessPlugins { +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +public abstract class AbstractTableRuntime extends PersistentBase implements TableRuntime { private final TableRuntimeStore store; + private final Map processContainerMap = Maps.newConcurrentMap(); protected AbstractTableRuntime(TableRuntimeStore store) { this.store = store; @@ -48,6 +57,44 @@ public TableConfiguration getTableConfiguration() { return TableConfigurations.parseTableConfig(store().getTableConfig()); } + @Override + public List getProcessStates() { + return processContainerMap.values().stream() + .flatMap(container -> container.getProcessStates().stream()) + .collect(Collectors.toList()); + } + + @Override + public List getProcessStates(Action action) { + return processContainerMap.get(action).getProcessStates(); + } + + @Override + public void registerProcess(TableProcessStore processStore) { + processContainerMap + .computeIfAbsent(processStore.getAction(), k -> new TableProcessContainer()) + .processLock + .lock(); + try { + processContainerMap + .get(processStore.getAction()) + .processMap + .put(processStore.getProcessId(), processStore); + } finally { + processContainerMap.get(processStore.getAction()).processLock.unlock(); + } + } + + @Override + public void removeProcess(TableProcessStore processStore) { + processContainerMap.computeIfPresent( + processStore.getAction(), + (action, container) -> { + container.processMap.remove(processStore.getProcessId()); + return container; + }); + } + @Override public String getGroupName() { return store().getGroupName(); @@ -61,4 +108,13 @@ public int getStatusCode() { public void dispose() { store().dispose(); } + + private static class TableProcessContainer { + private final Lock processLock = new ReentrantLock(); + private final Map processMap = Maps.newConcurrentMap(); + + public List getProcessStates() { + return Lists.newArrayList(processMap.values()); + } + } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/CompatibleTableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/CompatibleTableRuntime.java new file mode 100644 index 0000000000..85ecd4e054 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/CompatibleTableRuntime.java @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.table; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.api.BlockableOperation; +import org.apache.amoro.config.OptimizingConfig; +import org.apache.amoro.config.TableConfiguration; +import org.apache.amoro.iceberg.Constants; +import org.apache.amoro.metrics.MetricRegistry; +import org.apache.amoro.optimizing.OptimizingType; +import org.apache.amoro.optimizing.TableRuntimeOptimizingState; +import org.apache.amoro.optimizing.plan.AbstractOptimizingEvaluator; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.server.AmoroServiceConstants; +import org.apache.amoro.server.optimizing.OptimizingProcess; +import org.apache.amoro.server.optimizing.OptimizingStatus; +import org.apache.amoro.server.optimizing.TaskRuntime; +import org.apache.amoro.server.persistence.mapper.OptimizerMapper; +import org.apache.amoro.server.persistence.mapper.OptimizingProcessMapper; +import org.apache.amoro.server.persistence.mapper.TableBlockerMapper; +import org.apache.amoro.server.resource.OptimizerInstance; +import org.apache.amoro.server.table.blocker.TableBlocker; +import org.apache.amoro.server.table.cleanup.CleanupOperation; +import org.apache.amoro.server.table.cleanup.TableRuntimeCleanupState; +import org.apache.amoro.server.utils.IcebergTableUtil; +import org.apache.amoro.server.utils.SnowflakeIdGenerator; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.table.BaseTable; +import org.apache.amoro.table.ChangeTable; +import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.StateKey; +import org.apache.amoro.table.TableRuntimeStore; +import org.apache.amoro.table.UnkeyedTable; +import org.apache.iceberg.Snapshot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; + +/** Table runtime implementation compatible with old version optimizing process. */ +public class CompatibleTableRuntime extends DefaultTableRuntime implements TableRuntime { + + private static final Logger LOG = LoggerFactory.getLogger(CompatibleTableRuntime.class); + + private static final StateKey OPTIMIZING_STATE_KEY = + StateKey.stateKey("optimizing_state") + .jsonType(TableRuntimeOptimizingState.class) + .defaultValue(new TableRuntimeOptimizingState()); + + private static final StateKey PENDING_INPUT_KEY = + StateKey.stateKey("pending_input") + .jsonType(AbstractOptimizingEvaluator.PendingInput.class) + .defaultValue(new AbstractOptimizingEvaluator.PendingInput()); + + private static final StateKey CLEANUP_STATE_KEY = + StateKey.stateKey("cleanup_state") + .jsonType(TableRuntimeCleanupState.class) + .defaultValue(new TableRuntimeCleanupState()); + + private static final StateKey PROCESS_ID_KEY = + StateKey.stateKey("process_id").longType().defaultValue(0L); + + public static final List> REQUIRED_STATES = + Lists.newArrayList( + OPTIMIZING_STATE_KEY, PENDING_INPUT_KEY, PROCESS_ID_KEY, CLEANUP_STATE_KEY); + + private final TableOptimizingMetrics optimizingMetrics; + private final TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics; + private final TableSummaryMetrics tableSummaryMetrics; + private volatile long lastPlanTime; + private volatile OptimizingProcess optimizingProcess; + private final List taskQuotas = new CopyOnWriteArrayList<>(); + + public CompatibleTableRuntime(TableRuntimeStore store) { + super(store); + this.optimizingMetrics = + new TableOptimizingMetrics(store.getTableIdentifier(), store.getGroupName()); + this.orphanFilesCleaningMetrics = + new TableOrphanFilesCleaningMetrics(store.getTableIdentifier()); + this.tableSummaryMetrics = new TableSummaryMetrics(store.getTableIdentifier()); + } + + public void recover(OptimizingProcess optimizingProcess) { + if (!getOptimizingStatus().isProcessing() + || !Objects.equals(optimizingProcess.getProcessId(), getProcessId())) { + throw new IllegalStateException("Table runtime and processing are not matched!"); + } + this.optimizingProcess = optimizingProcess; + if (this.optimizingProcess.getStatus() == ProcessStatus.SUCCESS) { + completeProcess(true); + } + } + + @Override + public void registerMetric(MetricRegistry metricRegistry) { + // TODO: extract method to interface. + this.optimizingMetrics.register(metricRegistry); + this.orphanFilesCleaningMetrics.register(metricRegistry); + this.tableSummaryMetrics.register(metricRegistry); + } + + public TableOrphanFilesCleaningMetrics getOrphanFilesCleaningMetrics() { + return orphanFilesCleaningMetrics; + } + + public long getCurrentSnapshotId() { + return store().getState(OPTIMIZING_STATE_KEY).getCurrentSnapshotId(); + } + + public long getCurrentChangeSnapshotId() { + return store().getState(OPTIMIZING_STATE_KEY).getCurrentChangeSnapshotId(); + } + + public long getLastPlanTime() { + return lastPlanTime; + } + + public void setLastPlanTime(long lastPlanTime) { + this.lastPlanTime = lastPlanTime; + } + + public OptimizingStatus getOptimizingStatus() { + return OptimizingStatus.ofCode(getStatusCode()); + } + + public long getLastMajorOptimizingTime() { + return store().getState(OPTIMIZING_STATE_KEY).getLastMajorOptimizingTime(); + } + + public long getLastFullOptimizingTime() { + return store().getState(OPTIMIZING_STATE_KEY).getLastFullOptimizingTime(); + } + + public long getLastMinorOptimizingTime() { + return store().getState(OPTIMIZING_STATE_KEY).getLastMinorOptimizingTime(); + } + + public long getLastOptimizedChangeSnapshotId() { + return store().getState(OPTIMIZING_STATE_KEY).getLastOptimizedChangeSnapshotId(); + } + + public long getLastOptimizedSnapshotId() { + return store().getState(OPTIMIZING_STATE_KEY).getLastOptimizedSnapshotId(); + } + + public OptimizingConfig getOptimizingConfig() { + return getTableConfiguration().getOptimizingConfig(); + } + + public AbstractOptimizingEvaluator.PendingInput getPendingInput() { + return store().getState(PENDING_INPUT_KEY); + } + + public long getProcessId() { + return store().getState(PROCESS_ID_KEY); + } + + public OptimizingProcess getOptimizingProcess() { + return optimizingProcess; + } + + public void addTaskQuota(TaskRuntime.TaskQuota taskQuota) { + doAsIgnoreError(OptimizingProcessMapper.class, mapper -> mapper.insertTaskQuota(taskQuota)); + taskQuotas.add(taskQuota); + long validTime = System.currentTimeMillis() - AmoroServiceConstants.QUOTA_LOOK_BACK_TIME; + this.taskQuotas.removeIf(task -> task.checkExpired(validTime)); + } + + /** + * TODO: this is not final solution + * + * @param startTimeMills + */ + public void resetTaskQuotas(long startTimeMills) { + store() + .synchronizedInvoke( + () -> { + long minProcessId = SnowflakeIdGenerator.getMinSnowflakeId(startTimeMills); + taskQuotas.clear(); + taskQuotas.addAll( + getAs( + OptimizingProcessMapper.class, + mapper -> + mapper.selectTaskQuotasByTime( + getTableIdentifier().getId(), minProcessId))); + }); + } + + public double calculateQuotaOccupy() { + double targetQuota = getOptimizingConfig().getTargetQuota(); + int targetQuotaLimit = + targetQuota > 1 ? (int) targetQuota : (int) Math.ceil(targetQuota * getThreadCount()); + return (double) getQuotaTime() / AmoroServiceConstants.QUOTA_LOOK_BACK_TIME / targetQuotaLimit; + } + + public boolean isAllowPartialCommit() { + return getOptimizingConfig().isAllowPartialCommit(); + } + + public void setPendingInput(AbstractOptimizingEvaluator.PendingInput pendingInput) { + long pendingFileSize = + pendingInput.getDataFileSize() + + pendingInput.getEqualityDeleteBytes() + + pendingInput.getPositionalDeleteBytes(); + int pendingFileCount = + pendingInput.getDataFileCount() + + pendingInput.getEqualityDeleteFileCount() + + pendingInput.getPositionalDeleteFileCount(); + store() + .begin() + .updateState(PENDING_INPUT_KEY, i -> pendingInput) + .updateStatusCode( + code -> { + if (code == OptimizingStatus.IDLE.getCode()) { + LOG.info( + "{} status changed from idle to pending with pendingInput {}", + getTableIdentifier(), + pendingInput); + return OptimizingStatus.PENDING.getCode(); + } + return code; + }) + .updateTableSummary( + summary -> { + summary.setTotalFileSize(pendingFileSize); + summary.setTotalFileCount(pendingFileCount); + }) + .commit(); + } + + public void setTableSummary(AbstractOptimizingEvaluator.PendingInput tableSummary) { + store() + .begin() + .updateTableSummary( + summary -> { + summary.setHealthScore(tableSummary.getHealthScore()); + summary.setSmallFileScore(tableSummary.getSmallFileScore()); + summary.setEqualityDeleteScore(tableSummary.getEqualityDeleteScore()); + summary.setPositionalDeleteScore(tableSummary.getPositionalDeleteScore()); + summary.setTotalFileCount(tableSummary.getTotalFileCount()); + summary.setTotalFileSize(tableSummary.getTotalFileSize()); + }) + .commit(); + tableSummaryMetrics.refresh(tableSummary); + } + + public CompatibleTableRuntime refresh(AmoroTable table) { + Map tableConfig = table.properties(); + TableConfiguration newConfiguration = TableConfigurations.parseTableConfig(tableConfig); + String newGroupName = newConfiguration.getOptimizingConfig().getOptimizerGroup(); + + if (!Objects.equals(getGroupName(), newGroupName)) { + if (optimizingProcess != null) { + optimizingProcess.close(false); + } + this.optimizingMetrics.optimizerGroupChanged(getGroupName()); + } + + store() + .begin() + .updateTableConfig( + config -> { + config.clear(); + config.putAll(tableConfig); + }) + .updateGroup(g -> newGroupName) + .updateState( + OPTIMIZING_STATE_KEY, + s -> { + refreshSnapshots(table, s); + return s; + }) + .commit(); + return this; + } + + public void beginPlanning() { + OptimizingStatus originalStatus = getOptimizingStatus(); + store().begin().updateStatusCode(code -> OptimizingStatus.PLANNING.getCode()).commit(); + } + + public void planFailed() { + OptimizingStatus originalStatus = getOptimizingStatus(); + store().begin().updateStatusCode(code -> OptimizingStatus.PENDING.getCode()).commit(); + } + + public void beginProcess(OptimizingProcess optimizingProcess) { + OptimizingStatus originalStatus = getOptimizingStatus(); + this.optimizingProcess = optimizingProcess; + + store() + .begin() + .updateState(PROCESS_ID_KEY, any -> optimizingProcess.getProcessId()) + .updateStatusCode( + code -> + OptimizingStatus.ofOptimizingType(optimizingProcess.getOptimizingType()).getCode()) + .updateState(PENDING_INPUT_KEY, any -> new AbstractOptimizingEvaluator.PendingInput()) + .commit(); + } + + public long getLastCleanTime(CleanupOperation operation) { + TableRuntimeCleanupState state = store().getState(CLEANUP_STATE_KEY); + switch (operation) { + case ORPHAN_FILES_CLEANING: + return state.getLastOrphanFilesCleanTime(); + case DANGLING_DELETE_FILES_CLEANING: + return state.getLastDanglingDeleteFilesCleanTime(); + case DATA_EXPIRING: + return state.getLastDataExpiringTime(); + case SNAPSHOTS_EXPIRING: + return state.getLastSnapshotsExpiringTime(); + default: + return 0L; + } + } + + public void updateLastCleanTime(CleanupOperation operation, long time) { + store() + .begin() + .updateState( + CLEANUP_STATE_KEY, + state -> { + switch (operation) { + case ORPHAN_FILES_CLEANING: + state.setLastOrphanFilesCleanTime(time); + break; + case DANGLING_DELETE_FILES_CLEANING: + state.setLastDanglingDeleteFilesCleanTime(time); + break; + case DATA_EXPIRING: + state.setLastDataExpiringTime(time); + break; + case SNAPSHOTS_EXPIRING: + state.setLastSnapshotsExpiringTime(time); + break; + } + return state; + }) + .commit(); + } + + public void completeProcess(boolean success) { + OptimizingStatus originalStatus = getOptimizingStatus(); + OptimizingType processType = optimizingProcess.getOptimizingType(); + + store() + .begin() + .updateState( + OPTIMIZING_STATE_KEY, + state -> { + state.setLastOptimizedSnapshotId(optimizingProcess.getTargetSnapshotId()); + state.setLastOptimizedChangeSnapshotId(optimizingProcess.getTargetChangeSnapshotId()); + if (processType == OptimizingType.MINOR) { + state.setLastMinorOptimizingTime(optimizingProcess.getPlanTime()); + } else if (processType == OptimizingType.MAJOR) { + state.setLastMajorOptimizingTime(optimizingProcess.getPlanTime()); + } else if (processType == OptimizingType.FULL) { + state.setLastFullOptimizingTime(optimizingProcess.getPlanTime()); + } + return state; + }) + .updateStatusCode(code -> OptimizingStatus.IDLE.getCode()) + .commit(); + + optimizingMetrics.processComplete(processType, success, optimizingProcess.getPlanTime()); + optimizingProcess = null; + } + + public void completeEmptyProcess() { + OptimizingStatus originalStatus = getOptimizingStatus(); + boolean needUpdate = + originalStatus == OptimizingStatus.PLANNING || originalStatus == OptimizingStatus.PENDING; + if (needUpdate) { + store() + .begin() + .updateStatusCode(code -> OptimizingStatus.IDLE.getCode()) + .updateState( + OPTIMIZING_STATE_KEY, + state -> { + state.setLastOptimizedSnapshotId(state.getCurrentSnapshotId()); + state.setLastOptimizedChangeSnapshotId(state.getCurrentChangeSnapshotId()); + return state; + }) + .updateState(PENDING_INPUT_KEY, any -> new AbstractOptimizingEvaluator.PendingInput()) + .commit(); + } + } + + public void optimizingNotNecessary() { + if (getOptimizingStatus() == OptimizingStatus.IDLE) { + store() + .begin() + .updateState( + OPTIMIZING_STATE_KEY, + state -> { + state.setLastOptimizedSnapshotId(state.getCurrentSnapshotId()); + state.setLastOptimizedChangeSnapshotId(state.getCurrentChangeSnapshotId()); + return state; + }) + .commit(); + } + } + + public void beginCommitting() { + OptimizingStatus originalStatus = getOptimizingStatus(); + store().begin().updateStatusCode(code -> OptimizingStatus.COMMITTING.getCode()).commit(); + } + + @Override + public void unregisterMetric() { + tableSummaryMetrics.unregister(); + orphanFilesCleaningMetrics.unregister(); + optimizingMetrics.unregister(); + } + + @Override + public void dispose() { + unregisterMetric(); + store() + .synchronizedInvoke( + () -> { + Optional.ofNullable(optimizingProcess).ifPresent(process -> process.close(false)); + }); + super.dispose(); + } + + /** + * Check if operation are blocked now. + * + * @param operation - operation to check + * @return true if blocked + */ + public boolean isBlocked(BlockableOperation operation) { + List tableBlockers = + getAs( + TableBlockerMapper.class, + mapper -> + mapper.selectBlockers( + getTableIdentifier().getCatalog(), + getTableIdentifier().getDatabase(), + getTableIdentifier().getTableName(), + System.currentTimeMillis())); + return TableBlocker.conflict(operation, tableBlockers); + } + + private long getQuotaTime() { + long calculatingEndTime = System.currentTimeMillis(); + long calculatingStartTime = calculatingEndTime - AmoroServiceConstants.QUOTA_LOOK_BACK_TIME; + taskQuotas.removeIf(task -> task.checkExpired(calculatingStartTime)); + long finishedTaskQuotaTime = + taskQuotas.stream() + .mapToLong(taskQuota -> taskQuota.getQuotaTime(calculatingStartTime)) + .sum(); + return optimizingProcess == null + ? finishedTaskQuotaTime + : finishedTaskQuotaTime + + optimizingProcess.getRunningQuotaTime(calculatingStartTime, calculatingEndTime); + } + + private int getThreadCount() { + List instances = getAs(OptimizerMapper.class, OptimizerMapper::selectAll); + if (instances == null || instances.isEmpty()) { + return 1; + } + String groupName = getGroupName(); + return Math.max( + instances.stream() + .filter(instance -> Objects.equals(groupName, instance.getGroupName())) + .mapToInt(OptimizerInstance::getThreadCount) + .sum(), + 1); + } + + private boolean refreshSnapshots(AmoroTable amoroTable, TableRuntimeOptimizingState state) { + MixedTable table = (MixedTable) amoroTable.originalTable(); + tableSummaryMetrics.refreshSnapshots(table); + long lastSnapshotId = state.getCurrentSnapshotId(); + if (table.isKeyedTable()) { + long changeSnapshotId = state.getCurrentChangeSnapshotId(); + ChangeTable changeTable = table.asKeyedTable().changeTable(); + BaseTable baseTable = table.asKeyedTable().baseTable(); + + long currentChangeSnapshotId = doRefreshSnapshots(changeTable); + long currentSnapshotId = doRefreshSnapshots(baseTable); + + if (currentSnapshotId != lastSnapshotId || currentChangeSnapshotId != changeSnapshotId) { + LOG.debug( + "Refreshing table {} with base snapshot id {} and change snapshot id {}", + getTableIdentifier(), + currentSnapshotId, + currentChangeSnapshotId); + state.setCurrentChangeSnapshotId(currentChangeSnapshotId); + state.setCurrentSnapshotId(currentSnapshotId); + return true; + } + } else { + long currentSnapshotId = doRefreshSnapshots((UnkeyedTable) table); + if (currentSnapshotId != lastSnapshotId) { + LOG.debug( + "Refreshing table {} with base snapshot id {}", + getTableIdentifier(), + currentSnapshotId); + state.setCurrentSnapshotId(currentSnapshotId); + return true; + } + } + return false; + } + + private long doRefreshSnapshots(UnkeyedTable table) { + long currentSnapshotId = Constants.INVALID_SNAPSHOT_ID; + Snapshot currentSnapshot = IcebergTableUtil.getSnapshot(table, false); + if (currentSnapshot != null) { + currentSnapshotId = currentSnapshot.snapshotId(); + } + + optimizingMetrics.nonMaintainedSnapshotTime(currentSnapshot); + optimizingMetrics.lastOptimizingSnapshotTime( + IcebergTableUtil.findLatestOptimizingSnapshot(table).orElse(null)); + + return currentSnapshotId; + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultActionCoordinator.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultActionCoordinator.java new file mode 100644 index 0000000000..91dc07e05f --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultActionCoordinator.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.table; + +import org.apache.amoro.Action; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.process.ProcessTriggerStrategy; +import org.apache.amoro.process.TableProcess; +import org.apache.amoro.process.TableProcessStore; +import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; + +import java.util.Optional; + +public class DefaultActionCoordinator implements ActionCoordinator { + + private final Action action; + private final ProcessFactory factory; + private final TableFormat format; + private final ProcessTriggerStrategy strategy; + + public DefaultActionCoordinator(TableFormat format, Action action, ProcessFactory factory) { + this.action = action; + this.factory = factory; + this.format = format; + this.strategy = factory.triggerStrategy(format, action); + Preconditions.checkArgument( + strategy != null, "ProcessTriggerStrategy cannot be null for %s: %s", format, action); + } + + @Override + public boolean formatSupported(TableFormat format) { + return this.format.equals(format); + } + + @Override + public int parallelism() { + return strategy.getTriggerParallelism(); + } + + @Override + public Action action() { + return action; + } + + @Override + public long getNextExecutingTime(TableRuntime tableRuntime) { + return strategy.getTriggerInterval().toMillis(); + } + + @Override + public boolean enabled(TableRuntime tableRuntime) { + return formatSupported(tableRuntime.getFormat()); + } + + @Override + public long getExecutorDelay() { + return strategy.getTriggerInterval().toMillis(); + } + + @Override + public Optional trigger(TableRuntime tableRuntime) { + return factory.trigger(tableRuntime, action); + } + + @Override + public TableProcess recoverTableProcess( + TableRuntime tableRuntime, TableProcessStore processStore) { + return factory.recover(tableRuntime, processStore); + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java index 11c1a21cff..298e0e7ab9 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java @@ -18,598 +18,17 @@ package org.apache.amoro.server.table; -import org.apache.amoro.Action; -import org.apache.amoro.AmoroTable; -import org.apache.amoro.SupportsProcessPlugins; -import org.apache.amoro.TableRuntime; -import org.apache.amoro.api.BlockableOperation; -import org.apache.amoro.config.OptimizingConfig; -import org.apache.amoro.config.TableConfiguration; -import org.apache.amoro.iceberg.Constants; import org.apache.amoro.metrics.MetricRegistry; -import org.apache.amoro.optimizing.OptimizingType; -import org.apache.amoro.optimizing.TableRuntimeOptimizingState; -import org.apache.amoro.optimizing.plan.AbstractOptimizingEvaluator; -import org.apache.amoro.process.AmoroProcess; -import org.apache.amoro.process.ProcessFactory; -import org.apache.amoro.process.ProcessStatus; -import org.apache.amoro.process.TableProcessStore; -import org.apache.amoro.server.AmoroServiceConstants; -import org.apache.amoro.server.optimizing.OptimizingProcess; -import org.apache.amoro.server.optimizing.OptimizingStatus; -import org.apache.amoro.server.optimizing.TaskRuntime; -import org.apache.amoro.server.persistence.mapper.OptimizerMapper; -import org.apache.amoro.server.persistence.mapper.OptimizingProcessMapper; -import org.apache.amoro.server.persistence.mapper.TableBlockerMapper; -import org.apache.amoro.server.resource.OptimizerInstance; -import org.apache.amoro.server.table.blocker.TableBlocker; -import org.apache.amoro.server.table.cleanup.CleanupOperation; -import org.apache.amoro.server.table.cleanup.TableRuntimeCleanupState; -import org.apache.amoro.server.utils.IcebergTableUtil; -import org.apache.amoro.server.utils.SnowflakeIdGenerator; -import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; -import org.apache.amoro.shade.zookeeper3.org.apache.curator.shaded.com.google.common.collect.Maps; -import org.apache.amoro.table.BaseTable; -import org.apache.amoro.table.ChangeTable; -import org.apache.amoro.table.MixedTable; -import org.apache.amoro.table.StateKey; import org.apache.amoro.table.TableRuntimeStore; -import org.apache.amoro.table.UnkeyedTable; -import org.apache.iceberg.Snapshot; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -/** Default table runtime implementation. */ -public class DefaultTableRuntime extends AbstractTableRuntime - implements TableRuntime, SupportsProcessPlugins { - - private static final Logger LOG = LoggerFactory.getLogger(DefaultTableRuntime.class); - - private static final StateKey OPTIMIZING_STATE_KEY = - StateKey.stateKey("optimizing_state") - .jsonType(TableRuntimeOptimizingState.class) - .defaultValue(new TableRuntimeOptimizingState()); - - private static final StateKey PENDING_INPUT_KEY = - StateKey.stateKey("pending_input") - .jsonType(AbstractOptimizingEvaluator.PendingInput.class) - .defaultValue(new AbstractOptimizingEvaluator.PendingInput()); - - private static final StateKey CLEANUP_STATE_KEY = - StateKey.stateKey("cleanup_state") - .jsonType(TableRuntimeCleanupState.class) - .defaultValue(new TableRuntimeCleanupState()); - - private static final StateKey PROCESS_ID_KEY = - StateKey.stateKey("process_id").longType().defaultValue(0L); - - public static final List> REQUIRED_STATES = - Lists.newArrayList( - OPTIMIZING_STATE_KEY, PENDING_INPUT_KEY, PROCESS_ID_KEY, CLEANUP_STATE_KEY); - - private final Map processContainerMap = Maps.newConcurrentMap(); - private final TableOptimizingMetrics optimizingMetrics; - private final TableOrphanFilesCleaningMetrics orphanFilesCleaningMetrics; - private final TableSummaryMetrics tableSummaryMetrics; - private volatile long lastPlanTime; - private volatile OptimizingProcess optimizingProcess; - private final List taskQuotas = new CopyOnWriteArrayList<>(); - - public DefaultTableRuntime(TableRuntimeStore store) { +public class DefaultTableRuntime extends AbstractTableRuntime { + protected DefaultTableRuntime(TableRuntimeStore store) { super(store); - this.optimizingMetrics = - new TableOptimizingMetrics(store.getTableIdentifier(), store.getGroupName()); - this.orphanFilesCleaningMetrics = - new TableOrphanFilesCleaningMetrics(store.getTableIdentifier()); - this.tableSummaryMetrics = new TableSummaryMetrics(store.getTableIdentifier()); - } - - public void recover(OptimizingProcess optimizingProcess) { - if (!getOptimizingStatus().isProcessing() - || !Objects.equals(optimizingProcess.getProcessId(), getProcessId())) { - throw new IllegalStateException("Table runtime and processing are not matched!"); - } - this.optimizingProcess = optimizingProcess; - if (this.optimizingProcess.getStatus() == ProcessStatus.SUCCESS) { - completeProcess(true); - } - } - - @Override - public void registerMetric(MetricRegistry metricRegistry) { - // TODO: extract method to interface. - this.optimizingMetrics.register(metricRegistry); - this.orphanFilesCleaningMetrics.register(metricRegistry); - this.tableSummaryMetrics.register(metricRegistry); - } - - @Override - public AmoroProcess trigger(Action action) { - return Optional.ofNullable(processContainerMap.get(action)) - .map(container -> container.trigger(action)) - // Define a related exception - .orElseThrow(() -> new IllegalArgumentException("No ProcessFactory for action " + action)); - } - - @Override - public void install(Action action, ProcessFactory processFactory) { - if (processContainerMap.putIfAbsent(action, new TableProcessContainer(processFactory)) - != null) { - throw new IllegalStateException("ProcessFactory for action " + action + " already exists"); - } } @Override - public boolean enabled(Action action) { - return processContainerMap.get(action) != null; - } + public void registerMetric(MetricRegistry metricRegistry) {} @Override - public List getProcessStates() { - return processContainerMap.values().stream() - .flatMap(container -> container.getProcessStates().stream()) - .collect(Collectors.toList()); - } - - @Override - public List getProcessStates(Action action) { - return processContainerMap.get(action).getProcessStates(); - } - - public TableOrphanFilesCleaningMetrics getOrphanFilesCleaningMetrics() { - return orphanFilesCleaningMetrics; - } - - public long getCurrentSnapshotId() { - return store().getState(OPTIMIZING_STATE_KEY).getCurrentSnapshotId(); - } - - public long getCurrentChangeSnapshotId() { - return store().getState(OPTIMIZING_STATE_KEY).getCurrentChangeSnapshotId(); - } - - public long getLastPlanTime() { - return lastPlanTime; - } - - public void setLastPlanTime(long lastPlanTime) { - this.lastPlanTime = lastPlanTime; - } - - public OptimizingStatus getOptimizingStatus() { - return OptimizingStatus.ofCode(getStatusCode()); - } - - public long getLastMajorOptimizingTime() { - return store().getState(OPTIMIZING_STATE_KEY).getLastMajorOptimizingTime(); - } - - public long getLastFullOptimizingTime() { - return store().getState(OPTIMIZING_STATE_KEY).getLastFullOptimizingTime(); - } - - public long getLastMinorOptimizingTime() { - return store().getState(OPTIMIZING_STATE_KEY).getLastMinorOptimizingTime(); - } - - public long getLastOptimizedChangeSnapshotId() { - return store().getState(OPTIMIZING_STATE_KEY).getLastOptimizedChangeSnapshotId(); - } - - public long getLastOptimizedSnapshotId() { - return store().getState(OPTIMIZING_STATE_KEY).getLastOptimizedSnapshotId(); - } - - public OptimizingConfig getOptimizingConfig() { - return getTableConfiguration().getOptimizingConfig(); - } - - public AbstractOptimizingEvaluator.PendingInput getPendingInput() { - return store().getState(PENDING_INPUT_KEY); - } - - public long getProcessId() { - return store().getState(PROCESS_ID_KEY); - } - - public OptimizingProcess getOptimizingProcess() { - return optimizingProcess; - } - - public void addTaskQuota(TaskRuntime.TaskQuota taskQuota) { - doAsIgnoreError(OptimizingProcessMapper.class, mapper -> mapper.insertTaskQuota(taskQuota)); - taskQuotas.add(taskQuota); - long validTime = System.currentTimeMillis() - AmoroServiceConstants.QUOTA_LOOK_BACK_TIME; - this.taskQuotas.removeIf(task -> task.checkExpired(validTime)); - } - - /** - * TODO: this is not final solution - * - * @param startTimeMills - */ - public void resetTaskQuotas(long startTimeMills) { - store() - .synchronizedInvoke( - () -> { - long minProcessId = SnowflakeIdGenerator.getMinSnowflakeId(startTimeMills); - taskQuotas.clear(); - taskQuotas.addAll( - getAs( - OptimizingProcessMapper.class, - mapper -> - mapper.selectTaskQuotasByTime( - getTableIdentifier().getId(), minProcessId))); - }); - } - - public double calculateQuotaOccupy() { - double targetQuota = getOptimizingConfig().getTargetQuota(); - int targetQuotaLimit = - targetQuota > 1 ? (int) targetQuota : (int) Math.ceil(targetQuota * getThreadCount()); - return (double) getQuotaTime() / AmoroServiceConstants.QUOTA_LOOK_BACK_TIME / targetQuotaLimit; - } - - public boolean isAllowPartialCommit() { - return getOptimizingConfig().isAllowPartialCommit(); - } - - public void setPendingInput(AbstractOptimizingEvaluator.PendingInput pendingInput) { - long pendingFileSize = - pendingInput.getDataFileSize() - + pendingInput.getEqualityDeleteBytes() - + pendingInput.getPositionalDeleteBytes(); - int pendingFileCount = - pendingInput.getDataFileCount() - + pendingInput.getEqualityDeleteFileCount() - + pendingInput.getPositionalDeleteFileCount(); - store() - .begin() - .updateState(PENDING_INPUT_KEY, i -> pendingInput) - .updateStatusCode( - code -> { - if (code == OptimizingStatus.IDLE.getCode()) { - LOG.info( - "{} status changed from idle to pending with pendingInput {}", - getTableIdentifier(), - pendingInput); - return OptimizingStatus.PENDING.getCode(); - } - return code; - }) - .updateTableSummary( - summary -> { - summary.setTotalFileSize(pendingFileSize); - summary.setTotalFileCount(pendingFileCount); - }) - .commit(); - } - - public void setTableSummary(AbstractOptimizingEvaluator.PendingInput tableSummary) { - store() - .begin() - .updateTableSummary( - summary -> { - summary.setHealthScore(tableSummary.getHealthScore()); - summary.setSmallFileScore(tableSummary.getSmallFileScore()); - summary.setEqualityDeleteScore(tableSummary.getEqualityDeleteScore()); - summary.setPositionalDeleteScore(tableSummary.getPositionalDeleteScore()); - summary.setTotalFileCount(tableSummary.getTotalFileCount()); - summary.setTotalFileSize(tableSummary.getTotalFileSize()); - }) - .commit(); - tableSummaryMetrics.refresh(tableSummary); - } - - public DefaultTableRuntime refresh(AmoroTable table) { - Map tableConfig = table.properties(); - TableConfiguration newConfiguration = TableConfigurations.parseTableConfig(tableConfig); - String newGroupName = newConfiguration.getOptimizingConfig().getOptimizerGroup(); - - if (!Objects.equals(getGroupName(), newGroupName)) { - if (optimizingProcess != null) { - optimizingProcess.close(false); - } - this.optimizingMetrics.optimizerGroupChanged(getGroupName()); - } - - store() - .begin() - .updateTableConfig( - config -> { - config.clear(); - config.putAll(tableConfig); - }) - .updateGroup(g -> newGroupName) - .updateState( - OPTIMIZING_STATE_KEY, - s -> { - refreshSnapshots(table, s); - return s; - }) - .commit(); - return this; - } - - public void beginPlanning() { - OptimizingStatus originalStatus = getOptimizingStatus(); - store().begin().updateStatusCode(code -> OptimizingStatus.PLANNING.getCode()).commit(); - } - - public void planFailed() { - OptimizingStatus originalStatus = getOptimizingStatus(); - store().begin().updateStatusCode(code -> OptimizingStatus.PENDING.getCode()).commit(); - } - - public void beginProcess(OptimizingProcess optimizingProcess) { - OptimizingStatus originalStatus = getOptimizingStatus(); - this.optimizingProcess = optimizingProcess; - - store() - .begin() - .updateState(PROCESS_ID_KEY, any -> optimizingProcess.getProcessId()) - .updateStatusCode( - code -> - OptimizingStatus.ofOptimizingType(optimizingProcess.getOptimizingType()).getCode()) - .updateState(PENDING_INPUT_KEY, any -> new AbstractOptimizingEvaluator.PendingInput()) - .commit(); - } - - public long getLastCleanTime(CleanupOperation operation) { - TableRuntimeCleanupState state = store().getState(CLEANUP_STATE_KEY); - switch (operation) { - case ORPHAN_FILES_CLEANING: - return state.getLastOrphanFilesCleanTime(); - case DANGLING_DELETE_FILES_CLEANING: - return state.getLastDanglingDeleteFilesCleanTime(); - case DATA_EXPIRING: - return state.getLastDataExpiringTime(); - case SNAPSHOTS_EXPIRING: - return state.getLastSnapshotsExpiringTime(); - default: - return 0L; - } - } - - public void updateLastCleanTime(CleanupOperation operation, long time) { - store() - .begin() - .updateState( - CLEANUP_STATE_KEY, - state -> { - switch (operation) { - case ORPHAN_FILES_CLEANING: - state.setLastOrphanFilesCleanTime(time); - break; - case DANGLING_DELETE_FILES_CLEANING: - state.setLastDanglingDeleteFilesCleanTime(time); - break; - case DATA_EXPIRING: - state.setLastDataExpiringTime(time); - break; - case SNAPSHOTS_EXPIRING: - state.setLastSnapshotsExpiringTime(time); - break; - } - return state; - }) - .commit(); - } - - public void completeProcess(boolean success) { - OptimizingStatus originalStatus = getOptimizingStatus(); - OptimizingType processType = optimizingProcess.getOptimizingType(); - - store() - .begin() - .updateState( - OPTIMIZING_STATE_KEY, - state -> { - state.setLastOptimizedSnapshotId(optimizingProcess.getTargetSnapshotId()); - state.setLastOptimizedChangeSnapshotId(optimizingProcess.getTargetChangeSnapshotId()); - if (processType == OptimizingType.MINOR) { - state.setLastMinorOptimizingTime(optimizingProcess.getPlanTime()); - } else if (processType == OptimizingType.MAJOR) { - state.setLastMajorOptimizingTime(optimizingProcess.getPlanTime()); - } else if (processType == OptimizingType.FULL) { - state.setLastFullOptimizingTime(optimizingProcess.getPlanTime()); - } - return state; - }) - .updateStatusCode(code -> OptimizingStatus.IDLE.getCode()) - .commit(); - - optimizingMetrics.processComplete(processType, success, optimizingProcess.getPlanTime()); - optimizingProcess = null; - } - - public void completeEmptyProcess() { - OptimizingStatus originalStatus = getOptimizingStatus(); - boolean needUpdate = - originalStatus == OptimizingStatus.PLANNING || originalStatus == OptimizingStatus.PENDING; - if (needUpdate) { - store() - .begin() - .updateStatusCode(code -> OptimizingStatus.IDLE.getCode()) - .updateState( - OPTIMIZING_STATE_KEY, - state -> { - state.setLastOptimizedSnapshotId(state.getCurrentSnapshotId()); - state.setLastOptimizedChangeSnapshotId(state.getCurrentChangeSnapshotId()); - return state; - }) - .updateState(PENDING_INPUT_KEY, any -> new AbstractOptimizingEvaluator.PendingInput()) - .commit(); - } - } - - public void optimizingNotNecessary() { - if (getOptimizingStatus() == OptimizingStatus.IDLE) { - store() - .begin() - .updateState( - OPTIMIZING_STATE_KEY, - state -> { - state.setLastOptimizedSnapshotId(state.getCurrentSnapshotId()); - state.setLastOptimizedChangeSnapshotId(state.getCurrentChangeSnapshotId()); - return state; - }) - .commit(); - } - } - - public void beginCommitting() { - OptimizingStatus originalStatus = getOptimizingStatus(); - store().begin().updateStatusCode(code -> OptimizingStatus.COMMITTING.getCode()).commit(); - } - - @Override - public void unregisterMetric() { - tableSummaryMetrics.unregister(); - orphanFilesCleaningMetrics.unregister(); - optimizingMetrics.unregister(); - } - - @Override - public void dispose() { - unregisterMetric(); - store() - .synchronizedInvoke( - () -> { - Optional.ofNullable(optimizingProcess).ifPresent(process -> process.close(false)); - }); - super.dispose(); - } - - /** - * Check if operation are blocked now. - * - * @param operation - operation to check - * @return true if blocked - */ - public boolean isBlocked(BlockableOperation operation) { - List tableBlockers = - getAs( - TableBlockerMapper.class, - mapper -> - mapper.selectBlockers( - getTableIdentifier().getCatalog(), - getTableIdentifier().getDatabase(), - getTableIdentifier().getTableName(), - System.currentTimeMillis())); - return TableBlocker.conflict(operation, tableBlockers); - } - - private long getQuotaTime() { - long calculatingEndTime = System.currentTimeMillis(); - long calculatingStartTime = calculatingEndTime - AmoroServiceConstants.QUOTA_LOOK_BACK_TIME; - taskQuotas.removeIf(task -> task.checkExpired(calculatingStartTime)); - long finishedTaskQuotaTime = - taskQuotas.stream() - .mapToLong(taskQuota -> taskQuota.getQuotaTime(calculatingStartTime)) - .sum(); - return optimizingProcess == null - ? finishedTaskQuotaTime - : finishedTaskQuotaTime - + optimizingProcess.getRunningQuotaTime(calculatingStartTime, calculatingEndTime); - } - - private int getThreadCount() { - List instances = getAs(OptimizerMapper.class, OptimizerMapper::selectAll); - if (instances == null || instances.isEmpty()) { - return 1; - } - String groupName = getGroupName(); - return Math.max( - instances.stream() - .filter(instance -> Objects.equals(groupName, instance.getGroupName())) - .mapToInt(OptimizerInstance::getThreadCount) - .sum(), - 1); - } - - private boolean refreshSnapshots(AmoroTable amoroTable, TableRuntimeOptimizingState state) { - MixedTable table = (MixedTable) amoroTable.originalTable(); - tableSummaryMetrics.refreshSnapshots(table); - long lastSnapshotId = state.getCurrentSnapshotId(); - if (table.isKeyedTable()) { - long changeSnapshotId = state.getCurrentChangeSnapshotId(); - ChangeTable changeTable = table.asKeyedTable().changeTable(); - BaseTable baseTable = table.asKeyedTable().baseTable(); - - long currentChangeSnapshotId = doRefreshSnapshots(changeTable); - long currentSnapshotId = doRefreshSnapshots(baseTable); - - if (currentSnapshotId != lastSnapshotId || currentChangeSnapshotId != changeSnapshotId) { - LOG.debug( - "Refreshing table {} with base snapshot id {} and change snapshot id {}", - getTableIdentifier(), - currentSnapshotId, - currentChangeSnapshotId); - state.setCurrentChangeSnapshotId(currentChangeSnapshotId); - state.setCurrentSnapshotId(currentSnapshotId); - return true; - } - } else { - long currentSnapshotId = doRefreshSnapshots((UnkeyedTable) table); - if (currentSnapshotId != lastSnapshotId) { - LOG.debug( - "Refreshing table {} with base snapshot id {}", - getTableIdentifier(), - currentSnapshotId); - state.setCurrentSnapshotId(currentSnapshotId); - return true; - } - } - return false; - } - - private long doRefreshSnapshots(UnkeyedTable table) { - long currentSnapshotId = Constants.INVALID_SNAPSHOT_ID; - Snapshot currentSnapshot = IcebergTableUtil.getSnapshot(table, false); - if (currentSnapshot != null) { - currentSnapshotId = currentSnapshot.snapshotId(); - } - - optimizingMetrics.nonMaintainedSnapshotTime(currentSnapshot); - optimizingMetrics.lastOptimizingSnapshotTime( - IcebergTableUtil.findLatestOptimizingSnapshot(table).orElse(null)); - - return currentSnapshotId; - } - - private class TableProcessContainer { - private final Lock processLock = new ReentrantLock(); - private final ProcessFactory processFactory; - private final Map processMap = Maps.newConcurrentMap(); - - TableProcessContainer(ProcessFactory processFactory) { - this.processFactory = processFactory; - } - - public AmoroProcess trigger(Action action) { - processLock.lock(); - try { - AmoroProcess process = processFactory.create(DefaultTableRuntime.this, action); - process.getCompleteFuture().whenCompleted(() -> processMap.remove(process.getId())); - processMap.put(process.getId(), process); - return process; - } finally { - processLock.unlock(); - } - } - - public List getProcessStates() { - return processMap.values().stream().map(AmoroProcess::store).collect(Collectors.toList()); - } - } + public void unregisterMetric() {} } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntimeFactory.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntimeFactory.java index 86e748cc54..1b8d7ee323 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntimeFactory.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntimeFactory.java @@ -18,9 +18,16 @@ package org.apache.amoro.server.table; +import org.apache.amoro.Action; import org.apache.amoro.ServerTableIdentifier; import org.apache.amoro.TableFormat; import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.ProcessFactory; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; +import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; +import org.apache.amoro.shade.thrift.org.apache.commons.lang3.tuple.Pair; import org.apache.amoro.table.StateKey; import org.apache.amoro.table.TableRuntimeFactory; import org.apache.amoro.table.TableRuntimeStore; @@ -28,34 +35,91 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; public class DefaultTableRuntimeFactory implements TableRuntimeFactory { + private final List supportedCoordinators = Lists.newArrayList(); + + private final Map, ProcessFactory> actions = Maps.newHashMap(); + private final Set supportFormats = Sets.newHashSet(); + private final Set compatibleFormats = + Sets.newHashSet(TableFormat.MIXED_ICEBERG, TableFormat.MIXED_HIVE, TableFormat.ICEBERG); + @Override public void open(Map properties) {} @Override public void close() {} + public void initialize(List processFactories) { + processFactories.forEach( + f -> { + Map> supportedActions = f.supportedActions(); + for (TableFormat format : supportedActions.keySet()) { + Set supportedActionsForFormat = supportedActions.get(format); + for (Action action : supportedActionsForFormat) { + ProcessFactory exists = actions.get(Pair.of(format, action)); + if (exists != null) { + throw new IllegalArgumentException( + String.format( + "Plugin conflict, ProcessFactory: %s is supported for format: %s, action: %s, conflict with %s", + exists.name(), format, action, f.name())); + } + actions.put(Pair.of(format, action), f); + } + } + }); + actions.keySet().forEach(e -> supportFormats.add(e.getLeft())); + for (Pair key : actions.keySet()) { + supportedCoordinators.add( + new DefaultActionCoordinator(key.getLeft(), key.getRight(), actions.get(key))); + } + } + @Override - public String name() { - return "default"; + public List supportedCoordinators() { + return supportedCoordinators; } @Override public Optional accept( ServerTableIdentifier tableIdentifier, Map tableProperties) { - if (tableIdentifier - .getFormat() - .in(TableFormat.MIXED_ICEBERG, TableFormat.MIXED_HIVE, TableFormat.ICEBERG)) { - return Optional.of(new TableRuntimeCreatorImpl()); + if (!supportFormats.contains(tableIdentifier.getFormat())) { + if (!compatibleFormats.contains(tableIdentifier.getFormat())) { + return Optional.empty(); + } else { + return Optional.of(new CompatibleTableRuntimeCreatorImpl()); + } } - return Optional.empty(); + return Optional.of(new TableRuntimeCreatorImpl(tableIdentifier.getFormat())); } - private static class TableRuntimeCreatorImpl implements TableRuntimeFactory.TableRuntimeCreator { + @Override + public String name() { + return "default"; + } + + private class TableRuntimeCreatorImpl implements TableRuntimeFactory.TableRuntimeCreator { + + private final TableFormat format; + + public TableRuntimeCreatorImpl(TableFormat format) { + this.format = format; + } + @Override public List> requiredStateKeys() { - return DefaultTableRuntime.REQUIRED_STATES; + Map> requiredStates = Maps.newHashMap(); + actions.keySet().stream() + .filter(e -> e.getLeft().equals(format)) + .forEach( + e -> { + ProcessFactory factory = actions.get(e); + factory + .requiredStates() + .forEach(stateKey -> requiredStates.put(stateKey.getKey(), stateKey)); + }); + return Lists.newArrayList(requiredStates.values()); } @Override @@ -63,4 +127,18 @@ public TableRuntime create(TableRuntimeStore store) { return new DefaultTableRuntime(store); } } + + private static class CompatibleTableRuntimeCreatorImpl + implements TableRuntimeFactory.TableRuntimeCreator { + + @Override + public List> requiredStateKeys() { + return CompatibleTableRuntime.REQUIRED_STATES; + } + + @Override + public TableRuntime create(TableRuntimeStore store) { + return new CompatibleTableRuntime(store); + } + } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableService.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableService.java index 464cd5601d..6deeb340d9 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableService.java @@ -44,6 +44,7 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.amoro.shade.guava32.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.amoro.table.TableRuntimeFactory; import org.apache.amoro.table.TableSummary; import org.apache.amoro.utils.TablePropertyUtil; import org.slf4j.Logger; @@ -85,19 +86,19 @@ public class DefaultTableService extends PersistentBase implements TableService private final CompletableFuture initialized = new CompletableFuture<>(); private final Configurations serverConfiguration; private final CatalogManager catalogManager; - private final TableRuntimeFactoryManager tableRuntimeFactoryManager; + private final List tableRuntimeFactoryList; private RuntimeHandlerChain headHandler; private ExecutorService tableExplorerExecutors; public DefaultTableService( Configurations configuration, CatalogManager catalogManager, - TableRuntimeFactoryManager tableRuntimeFactoryManager) { + List tableRuntimeFactoryList) { this.catalogManager = catalogManager; this.externalCatalogRefreshingInterval = configuration.get(AmoroManagementConf.REFRESH_EXTERNAL_CATALOGS_INTERVAL).toMillis(); this.serverConfiguration = configuration; - this.tableRuntimeFactoryManager = tableRuntimeFactoryManager; + this.tableRuntimeFactoryList = tableRuntimeFactoryList; } @Override @@ -515,7 +516,7 @@ private Optional createTableRuntime( ServerTableIdentifier identifier, TableRuntimeMeta runtimeMeta, List restoredStates) { - return tableRuntimeFactoryManager.installedPlugins().stream() + return tableRuntimeFactoryList.stream() .map(f -> f.accept(identifier, runtimeMeta.getTableConfig())) .filter(Optional::isPresent) .map(Optional::get) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java b/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java index 0b96dddd80..7dc5cfa8e4 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/utils/IcebergTableUtil.java @@ -33,7 +33,7 @@ import org.apache.amoro.optimizing.plan.MixedIcebergOptimizingEvaluator; import org.apache.amoro.optimizing.plan.MixedIcebergOptimizingPlanner; import org.apache.amoro.scan.TableEntriesScan; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; import org.apache.amoro.shade.guava32.com.google.common.base.Predicate; import org.apache.amoro.shade.guava32.com.google.common.collect.Iterables; @@ -89,7 +89,8 @@ public static long getSnapshotId(Table table, boolean refresh) { } } - public static TableSnapshot getSnapshot(MixedTable mixedTable, DefaultTableRuntime tableRuntime) { + public static TableSnapshot getSnapshot( + MixedTable mixedTable, CompatibleTableRuntime tableRuntime) { if (mixedTable.isUnkeyedTable()) { return new BasicTableSnapshot(tableRuntime.getCurrentSnapshotId()); } else { @@ -220,7 +221,7 @@ public static Set getAllManifestFiles(Table table) { } public static AbstractOptimizingEvaluator createOptimizingEvaluator( - DefaultTableRuntime tableRuntime, + CompatibleTableRuntime tableRuntime, MixedTable table, TableSnapshot snapshot, int maxPendingPartitions) { @@ -264,13 +265,13 @@ public static AbstractOptimizingEvaluator createOptimizingEvaluator( } public static AbstractOptimizingEvaluator createOptimizingEvaluator( - DefaultTableRuntime tableRuntime, MixedTable table, int maxPendingPartitions) { + CompatibleTableRuntime tableRuntime, MixedTable table, int maxPendingPartitions) { TableSnapshot snapshot = IcebergTableUtil.getSnapshot(table, tableRuntime); return createOptimizingEvaluator(tableRuntime, table, snapshot, maxPendingPartitions); } public static AbstractOptimizingPlanner createOptimizingPlanner( - DefaultTableRuntime tableRuntime, + CompatibleTableRuntime tableRuntime, MixedTable table, double availableCore, long maxInputSizePerThread) { diff --git a/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql b/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql index f1f91e3a32..e142a2ce95 100644 --- a/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql +++ b/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql @@ -189,20 +189,6 @@ CREATE TABLE `task_runtime` KEY `table_index` (`table_id`, `process_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize task basic information'; -CREATE TABLE `table_process_state` -( - `process_id` bigint(20) NOT NULL COMMENT 'optimizing_procedure UUID', - `action` varchar(16) NOT NULL COMMENT 'process action', - `table_id` bigint(20) NOT NULL, - `retry_num` int(11) DEFAULT NULL COMMENT 'Retry times', - `status` varchar(10) NOT NULL COMMENT 'Direct to TableOptimizingStatus', - `start_time` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT 'First plan time', - `end_time` timestamp NULL DEFAULT NULL COMMENT 'finish time or failed time', - `fail_reason` varchar(4096) DEFAULT NULL COMMENT 'Error message after task failed', - `summary` mediumtext COMMENT 'state summary, usually a map', - PRIMARY KEY (`process_id`), - KEY `table_index` (`table_id`, `start_time`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'History of optimizing after each commit'; CREATE TABLE `optimizing_task_quota` ( diff --git a/amoro-ams/src/main/resources/mysql/upgrade.sql b/amoro-ams/src/main/resources/mysql/upgrade.sql index b36064109f..dca7839efc 100644 --- a/amoro-ams/src/main/resources/mysql/upgrade.sql +++ b/amoro-ams/src/main/resources/mysql/upgrade.sql @@ -107,18 +107,6 @@ CREATE TABLE `table_runtime` ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Table running information of each table' ROW_FORMAT=DYNAMIC; -CREATE TABLE `table_runtime_state` ( - `state_id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'Primary key', - `table_id` bigint unsigned NOT NULL COMMENT 'Table identifier id', - `state_key` varchar(256) NOT NULL COMMENT 'Table Runtime state key', - `state_value` mediumtext COMMENT 'Table Runtime state value, string type', - `state_version` bigint NOT NULL DEFAULT '0' COMMENT 'Table runtime state version, auto inc when update', - `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', - PRIMARY KEY (`state_id`), - UNIQUE KEY `uniq_table_state_key` (`table_id`,`state_key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='State of Table Runtimes'; - INSERT INTO table_runtime (`table_id`, `group_name`, `status_code`, `status_code_update_time`, `table_config`, `table_summary`) SELECT `table_id`, `optimizer_group`, `optimizing_status_code`, `optimizing_status_start_time`, diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java index 9f3d25b3f7..ea6dea031d 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/AMSServiceTestBase.java @@ -19,11 +19,13 @@ package org.apache.amoro.server; import org.apache.amoro.config.Configurations; +import org.apache.amoro.process.ActionCoordinator; import org.apache.amoro.resource.ResourceGroup; import org.apache.amoro.server.manager.EventsManager; import org.apache.amoro.server.manager.MetricManager; import org.apache.amoro.server.process.ProcessService; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.process.executor.ExecuteEngineManager; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.DefaultTableRuntimeFactory; import org.apache.amoro.server.table.DefaultTableService; import org.apache.amoro.server.table.TableRuntimeFactoryManager; @@ -34,6 +36,8 @@ import org.mockito.Mockito; import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; public abstract class AMSServiceTestBase extends AMSManagerTestBase { private static DefaultTableService TABLE_SERVICE = null; @@ -48,6 +52,10 @@ public static void initTableService() { tableRuntimeFactoryManager = Mockito.mock(TableRuntimeFactoryManager.class); Mockito.when(tableRuntimeFactoryManager.installedPlugins()) .thenReturn(Lists.newArrayList(runtimeFactory)); + List actionCoordinators = + tableRuntimeFactoryManager.installedPlugins().stream() + .flatMap(factory -> factory.supportedCoordinators().stream()) + .collect(Collectors.toList()); try { Configurations configurations = new Configurations(); configurations.set(AmoroManagementConf.OPTIMIZER_HB_TIMEOUT, Duration.ofMillis(800L)); @@ -55,11 +63,13 @@ public static void initTableService() { AmoroManagementConf.OPTIMIZER_TASK_EXECUTE_TIMEOUT, Duration.ofMillis(30000L)); TABLE_SERVICE = new DefaultTableService( - new Configurations(), CATALOG_MANAGER, tableRuntimeFactoryManager); + new Configurations(), CATALOG_MANAGER, Lists.newArrayList(runtimeFactory)); OPTIMIZING_SERVICE = new DefaultOptimizingService( configurations, CATALOG_MANAGER, OPTIMIZER_MANAGER, TABLE_SERVICE); - PROCESS_SERVICE = new ProcessService(configurations, TABLE_SERVICE); + PROCESS_SERVICE = + new ProcessService( + configurations, TABLE_SERVICE, actionCoordinators, new ExecuteEngineManager()); TABLE_SERVICE.addHandlerChain(OPTIMIZING_SERVICE.getTableRuntimeHandler()); TABLE_SERVICE.addHandlerChain(PROCESS_SERVICE.getTableHandlerChain()); @@ -87,8 +97,8 @@ protected DefaultTableService tableService() { return TABLE_SERVICE; } - protected DefaultTableRuntime getDefaultTableRuntime(long tableId) { - return (DefaultTableRuntime) tableService().getRuntime(tableId); + protected CompatibleTableRuntime getDefaultTableRuntime(long tableId) { + return (CompatibleTableRuntime) tableService().getRuntime(tableId); } protected DefaultOptimizingService optimizingService() { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java index f9fa8ce94d..88755fcc4e 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java @@ -39,7 +39,7 @@ import org.apache.amoro.server.resource.OptimizerInstance; import org.apache.amoro.server.scheduler.inline.TableRuntimeRefreshExecutor; import org.apache.amoro.server.table.AMSTableTestBase; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.MixedTable; @@ -114,7 +114,7 @@ private void initTableWithFiles() { (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); appendData(mixedTable.asUnkeyedTable(), 1); appendData(mixedTable.asUnkeyedTable(), 2); - DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); runtime.refresh(tableService().loadTable(serverTableIdentifier())); } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewManager.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewManager.java index 2b40652c2d..caa088a574 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewManager.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewManager.java @@ -33,7 +33,7 @@ import org.apache.amoro.server.dashboard.model.OverviewTopTableItem; import org.apache.amoro.server.scheduler.inline.TableRuntimeRefreshExecutor; import org.apache.amoro.server.table.AMSTableTestBase; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.UnkeyedTable; @@ -91,7 +91,7 @@ private void initTableWithFiles() { .asUnkeyedTable(); appendData(table, 1); appendData(table, 2); - DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); runtime.refresh(tableService().loadTable(serverTableIdentifier())); } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/utils/TestOptimizingUtil.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/utils/TestOptimizingUtil.java index b7b7bed034..769017d20c 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/utils/TestOptimizingUtil.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/utils/TestOptimizingUtil.java @@ -37,7 +37,7 @@ import org.apache.amoro.server.resource.OptimizerThread; import org.apache.amoro.server.resource.QuotaProvider; import org.apache.amoro.server.table.AMSTableTestBase; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.UnkeyedTable; @@ -90,7 +90,7 @@ public void testCalculateQuotaOccupy() { Assertions.assertEquals(0, OptimizingUtil.calculateQuotaOccupy(null, null, startTime, endTime)); - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); Assert.assertEquals(0, queue.collectTasks().size()); TaskRuntime task = queue.pollTask(optimizerThread, MAX_POLLING_TIME); @@ -126,7 +126,7 @@ public void testCalculateQuotaOccupy() { > OptimizingUtil.calculateQuotaOccupy(tasks, null, startTime, endTime)); } - protected OptimizingQueue buildOptimizingGroupService(DefaultTableRuntime tableRuntime) { + protected OptimizingQueue buildOptimizingGroupService(CompatibleTableRuntime tableRuntime) { return new OptimizingQueue( CATALOG_MANAGER, testResourceGroup(), @@ -140,12 +140,12 @@ protected static ResourceGroup testResourceGroup() { return new ResourceGroup.Builder("test", "local").build(); } - protected DefaultTableRuntime initTableWithFiles() { + protected CompatibleTableRuntime initTableWithFiles() { MixedTable mixedTable = (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); appendData(mixedTable.asUnkeyedTable(), 1); appendData(mixedTable.asUnkeyedTable(), 2); - DefaultTableRuntime tableRuntime = + CompatibleTableRuntime tableRuntime = buildTableRuntimeMeta(OptimizingStatus.PENDING, defaultResourceGroup()); tableRuntime.refresh(tableService().loadTable(serverTableIdentifier())); @@ -163,12 +163,12 @@ private void appendData(UnkeyedTable table, int id) { appendFiles.commit(); } - private DefaultTableRuntime buildTableRuntimeMeta( + private CompatibleTableRuntime buildTableRuntimeMeta( OptimizingStatus status, ResourceGroup resourceGroup) { MixedTable mixedTable = (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); - DefaultTableRuntime tableRuntime = - (DefaultTableRuntime) tableService().getRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime tableRuntime = + (CompatibleTableRuntime) tableService().getRuntime(serverTableIdentifier().getId()); tableRuntime .store() .begin() diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java index 342b533aaa..6b4c8e113f 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java @@ -52,7 +52,7 @@ import org.apache.amoro.server.resource.OptimizerThread; import org.apache.amoro.server.resource.QuotaProvider; import org.apache.amoro.server.table.AMSTableTestBase; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; @@ -106,7 +106,7 @@ protected static ResourceGroup testResourceGroup() { return new ResourceGroup.Builder("test", "local").build(); } - protected OptimizingQueue buildOptimizingGroupService(DefaultTableRuntime tableRuntime) { + protected OptimizingQueue buildOptimizingGroupService(CompatibleTableRuntime tableRuntime) { return new OptimizingQueue( CATALOG_MANAGER, testResourceGroup(), @@ -128,7 +128,7 @@ private OptimizingQueue buildOptimizingGroupService() { @Test public void testPollNoTask() { - DefaultTableRuntime tableRuntimeMeta = + CompatibleTableRuntime tableRuntimeMeta = buildTableRuntimeMeta(OptimizingStatus.PENDING, defaultResourceGroup()); OptimizingQueue queue = buildOptimizingGroupService(tableRuntimeMeta); Assert.assertNull(queue.pollTask(optimizerThread, 0)); @@ -139,7 +139,7 @@ public void testPollNoTask() { public void testRefreshAndReleaseTable() { OptimizingQueue queue = buildOptimizingGroupService(); Assert.assertEquals(0, queue.getSchedulingPolicy().getTableRuntimeMap().size()); - DefaultTableRuntime tableRuntime = + CompatibleTableRuntime tableRuntime = buildTableRuntimeMeta(OptimizingStatus.IDLE, defaultResourceGroup()); queue.refreshTable(tableRuntime); Assert.assertEquals(1, queue.getSchedulingPolicy().getTableRuntimeMap().size()); @@ -156,7 +156,7 @@ public void testRefreshAndReleaseTable() { @Test public void testPollTask() { - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); // 1.poll task @@ -170,7 +170,7 @@ public void testPollTask() { @Test public void testPollTaskWithOverQuotaDisabled() { - DefaultTableRuntime tableRuntime = initTableWithPartitionedFiles(); + CompatibleTableRuntime tableRuntime = initTableWithPartitionedFiles(); OptimizingQueue queue = new OptimizingQueue( CATALOG_MANAGER, @@ -204,7 +204,7 @@ public void testPollTaskWithOverQuotaDisabled() { @Test public void testPollTaskWithOverQuotaEnabled() { - DefaultTableRuntime tableRuntime = initTableWithPartitionedFiles(); + CompatibleTableRuntime tableRuntime = initTableWithPartitionedFiles(); OptimizingQueue queue = new OptimizingQueue( CATALOG_MANAGER, @@ -238,7 +238,7 @@ public void testPollTaskWithOverQuotaEnabled() { @Test public void testQuotaSchedulePolicy() throws InterruptedException { - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = new OptimizingQueue( @@ -273,7 +273,7 @@ public void testQuotaSchedulePolicy() throws InterruptedException { serverTableIdentifier().getCatalog(), "db", "new_table"), TableFormat.ICEBERG); serverTableIdentifier.setId(100L); - DefaultTableRuntime tableRuntime2 = createTable(serverTableIdentifier); + CompatibleTableRuntime tableRuntime2 = createTable(serverTableIdentifier); queue.refreshTable(tableRuntime2); queue.refreshTable(tableRuntime); @@ -289,7 +289,7 @@ public void testQuotaSchedulePolicy() throws InterruptedException { @Test public void testRetryTask() { - DefaultTableRuntime tableRuntimeMeta = initTableWithFiles(); + CompatibleTableRuntime tableRuntimeMeta = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntimeMeta); // 1.poll task @@ -320,7 +320,7 @@ public void testRetryTask() { @Test public void testCommitTask() { - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); Assert.assertEquals(0, queue.collectTasks().size()); @@ -351,7 +351,7 @@ public void testCommitTask() { @Test public void testCommitTaskWithFailed() { - DefaultTableRuntime tableRuntime = initTableWithPartitionedFiles(); + CompatibleTableRuntime tableRuntime = initTableWithPartitionedFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); Assert.assertEquals(0, queue.collectTasks().size()); @@ -399,7 +399,7 @@ public void testCommitTaskWithFailed() { @Test public void testCollectingTasks() { - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); Assert.assertEquals(0, queue.collectTasks().size()); @@ -413,7 +413,7 @@ public void testCollectingTasks() { @Test public void testTaskAndTableMetrics() { - DefaultTableRuntime tableRuntime = initTableWithFiles(); + CompatibleTableRuntime tableRuntime = initTableWithFiles(); OptimizingQueue queue = buildOptimizingGroupService(tableRuntime); MetricRegistry registry = MetricManager.getInstance().getGlobalRegistry(); Map tagValues = ImmutableMap.of(GROUP_TAG, testResourceGroup().getName()); @@ -526,7 +526,7 @@ public void testAddAndRemoveOptimizers() { queue.dispose(); } - protected DefaultTableRuntime initTableWithFiles() { + protected CompatibleTableRuntime initTableWithFiles() { MixedTable mixedTable = (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); mixedTable @@ -536,29 +536,29 @@ protected DefaultTableRuntime initTableWithFiles() { .commit(); appendData(mixedTable.asUnkeyedTable(), 1); appendData(mixedTable.asUnkeyedTable(), 2); - DefaultTableRuntime tableRuntime = + CompatibleTableRuntime tableRuntime = buildTableRuntimeMeta(OptimizingStatus.PENDING, defaultResourceGroup()); tableRuntime.refresh(tableService().loadTable(serverTableIdentifier())); return tableRuntime; } - protected DefaultTableRuntime initTableWithPartitionedFiles() { + protected CompatibleTableRuntime initTableWithPartitionedFiles() { MixedTable mixedTable = (MixedTable) tableService().loadTable(serverTableIdentifier()).originalTable(); appendPartitionedData(mixedTable.asUnkeyedTable(), 1); appendPartitionedData(mixedTable.asUnkeyedTable(), 2); - DefaultTableRuntime tableRuntime = + CompatibleTableRuntime tableRuntime = buildTableRuntimeMeta(OptimizingStatus.PENDING, defaultResourceGroup()); tableRuntime.refresh(tableService().loadTable(serverTableIdentifier())); return tableRuntime; } - private DefaultTableRuntime buildTableRuntimeMeta( + private CompatibleTableRuntime buildTableRuntimeMeta( OptimizingStatus status, ResourceGroup resourceGroup) { - DefaultTableRuntime tableRuntime = - (DefaultTableRuntime) tableService().getRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime tableRuntime = + (CompatibleTableRuntime) tableService().getRuntime(serverTableIdentifier().getId()); tableRuntime .store() .begin() @@ -594,7 +594,7 @@ private void appendData(UnkeyedTable table, int id) { appendFiles.commit(); } - private DefaultTableRuntime createTable(ServerTableIdentifier serverTableIdentifier) { + private CompatibleTableRuntime createTable(ServerTableIdentifier serverTableIdentifier) { org.apache.iceberg.catalog.Catalog catalog = catalogTestHelper().buildIcebergCatalog(catalogMeta()); catalog.createTable( @@ -613,8 +613,8 @@ private DefaultTableRuntime createTable(ServerTableIdentifier serverTableIdentif appendPartitionedData(mixedTable.asUnkeyedTable(), 1); appendPartitionedData(mixedTable.asUnkeyedTable(), 2); - DefaultTableRuntime tableRuntime = - (DefaultTableRuntime) tableService().getRuntime(serverTableIdentifier.getId()); + CompatibleTableRuntime tableRuntime = + (CompatibleTableRuntime) tableService().getRuntime(serverTableIdentifier.getId()); tableRuntime .store() diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/CompleteOptimizingFlow.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/CompleteOptimizingFlow.java index f26773395e..b92d9f631c 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/CompleteOptimizingFlow.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/CompleteOptimizingFlow.java @@ -38,7 +38,7 @@ import org.apache.amoro.server.optimizing.KeyedTableCommit; import org.apache.amoro.server.optimizing.TaskRuntime; import org.apache.amoro.server.optimizing.UnKeyedTableCommit; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableConfigurations; import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.table.MixedTable; @@ -181,7 +181,7 @@ private List> mockTaskRuntime( private AbstractOptimizingPlanner planner() { table.refresh(); - DefaultTableRuntime tableRuntime = Mockito.mock(DefaultTableRuntime.class); + CompatibleTableRuntime tableRuntime = Mockito.mock(CompatibleTableRuntime.class); Mockito.when(tableRuntime.getCurrentSnapshotId()).thenAnswer(f -> getCurrentSnapshotId()); Mockito.when(tableRuntime.getCurrentChangeSnapshotId()) diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java index 679aa46122..d651d53094 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java @@ -31,7 +31,7 @@ import org.apache.amoro.optimizing.plan.AbstractPartitionPlan; import org.apache.amoro.optimizing.scan.TableFileScanHelper; import org.apache.amoro.server.optimizing.OptimizingTestHelpers; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableConfigurations; import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; @@ -66,7 +66,7 @@ public abstract class MixedTablePlanTestBase extends TableTestBase { - protected DefaultTableRuntime tableRuntime; + protected CompatibleTableRuntime tableRuntime; public MixedTablePlanTestBase( CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { @@ -75,7 +75,7 @@ public MixedTablePlanTestBase( @Before public void mock() { - tableRuntime = Mockito.mock(DefaultTableRuntime.class); + tableRuntime = Mockito.mock(CompatibleTableRuntime.class); ServerTableIdentifier id = ServerTableIdentifier.of(getMixedTable().id(), getTestFormat()); id.setId(0L); Mockito.when(tableRuntime.getTableIdentifier()).thenReturn(id); @@ -555,7 +555,7 @@ protected long beginTransaction() { } } - protected DefaultTableRuntime getTableRuntime() { + protected CompatibleTableRuntime getTableRuntime() { return tableRuntime; } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestIcebergPartitionPlan.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestIcebergPartitionPlan.java index 7ad399709b..18e365409d 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestIcebergPartitionPlan.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestIcebergPartitionPlan.java @@ -29,7 +29,7 @@ import org.apache.amoro.optimizing.plan.IcebergPartitionPlan; import org.apache.amoro.optimizing.scan.IcebergTableFileScanHelper; import org.apache.amoro.optimizing.scan.TableFileScanHelper; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.junit.Test; @@ -75,7 +75,7 @@ public void testOnlyOneFragmentFiles() { @Override protected AbstractPartitionPlan getPartitionPlan() { - DefaultTableRuntime tableRuntime = getTableRuntime(); + CompatibleTableRuntime tableRuntime = getTableRuntime(); return new IcebergPartitionPlan( tableRuntime.getTableIdentifier(), tableRuntime.getOptimizingConfig(), diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/process/MockActionCoordinator.java b/amoro-ams/src/test/java/org/apache/amoro/server/process/MockActionCoordinator.java index da3617a148..f3e16ef834 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/process/MockActionCoordinator.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/process/MockActionCoordinator.java @@ -21,18 +21,21 @@ import org.apache.amoro.Action; import org.apache.amoro.TableFormat; import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.ActionCoordinator; import org.apache.amoro.process.TableProcess; import org.apache.amoro.process.TableProcessStore; +import org.apache.amoro.server.utils.SnowflakeIdGenerator; import java.util.HashMap; -import java.util.Map; +import java.util.Optional; /** Mock implementation of {@link ActionCoordinator} used in tests. */ public class MockActionCoordinator implements ActionCoordinator { public static final int PROCESS_MAX_POOL_SIZE = 1000; private static final TableFormat[] DEFAULT_FORMATS = new TableFormat[] {TableFormat.PAIMON}; + SnowflakeIdGenerator SNOWFLAKE_ID_GENERATOR = new SnowflakeIdGenerator(); - public static final Action DEFAULT_ACTION = new Action(DEFAULT_FORMATS, 0, "default_action"); + public static final Action DEFAULT_ACTION = Action.register("default_action"); /** * Whether the format is supported. @@ -61,12 +64,6 @@ public Action action() { return DEFAULT_ACTION; } - /** Get execution engine name. */ - @Override - public String executionEngine() { - return "default"; - } - /** Next executing time. */ @Override public long getNextExecutingTime(TableRuntime tableRuntime) { @@ -92,19 +89,19 @@ public long getExecutorDelay() { * @return mock process */ @Override - public TableProcess createTableProcess(TableRuntime tableRuntime) { + public Optional trigger(TableRuntime tableRuntime) { TableProcessMeta tableProcessMeta = TableProcessMeta.of( SNOWFLAKE_ID_GENERATOR.generateId(), tableRuntime.getTableIdentifier().getId(), action().getName(), - executionEngine(), + "default", new HashMap<>()); TableProcessStore tableProcessStore = new DefaultTableProcessStore( tableProcessMeta.getProcessId(), tableRuntime, tableProcessMeta, action(), 3); MockTableProcess mockTableProcess = new MockTableProcess(tableRuntime, tableProcessStore); - return mockTableProcess; + return Optional.of(mockTableProcess); } /** @@ -119,30 +116,4 @@ public TableProcess recoverTableProcess( TableRuntime tableRuntime, TableProcessStore processStore) { return new MockTableProcess(tableRuntime, processStore); } - - /** Return same process to cancel. */ - @Override - public TableProcess cancelTableProcess(TableRuntime tableRuntime, TableProcess process) { - return process; - } - - /** Return same process to retry. */ - @Override - public TableProcess retryTableProcess(TableProcess process) { - return process; - } - - /** Open plugin. */ - @Override - public void open(Map properties) {} - - /** Close plugin. */ - @Override - public void close() {} - - /** Plugin name. */ - @Override - public String name() { - return "mock_action_coordinator"; - } } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestBlockerExpiringExecutor.java b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestBlockerExpiringExecutor.java index eb79622b54..3adbe08308 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestBlockerExpiringExecutor.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestBlockerExpiringExecutor.java @@ -24,7 +24,7 @@ import org.apache.amoro.server.AMSServiceTestBase; import org.apache.amoro.server.persistence.PersistentBase; import org.apache.amoro.server.persistence.mapper.TableBlockerMapper; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.TableService; import org.apache.amoro.server.table.blocker.TableBlocker; import org.junit.Assert; @@ -41,12 +41,12 @@ public class TestBlockerExpiringExecutor extends AMSServiceTestBase { 0L, "test_catalog", "test_db", "test_table_blocker", TableFormat.MIXED_ICEBERG); private final Persistency persistency = new Persistency(); - private DefaultTableRuntime tableRuntime; + private CompatibleTableRuntime tableRuntime; private TableService tableService; @Before public void mock() { - tableRuntime = Mockito.mock(DefaultTableRuntime.class); + tableRuntime = Mockito.mock(CompatibleTableRuntime.class); tableService = Mockito.mock(TableService.class); Mockito.when(tableRuntime.getTableIdentifier()).thenReturn(tableIdentifier); } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java index c401c88d8c..af0f0934e8 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java @@ -26,7 +26,7 @@ import org.apache.amoro.server.persistence.TableRuntimeMeta; import org.apache.amoro.server.persistence.mapper.TableMetaMapper; import org.apache.amoro.server.persistence.mapper.TableRuntimeMapper; -import org.apache.amoro.server.table.DefaultTableRuntime; +import org.apache.amoro.server.table.CompatibleTableRuntime; import org.apache.amoro.server.table.DefaultTableRuntimeStore; import org.apache.amoro.server.table.TableRuntimeHandler; import org.apache.amoro.server.table.cleanup.CleanupOperation; @@ -81,12 +81,12 @@ private ServerTableIdentifier createTableIdentifier(long tableId) { } /** - * Create a test DefaultTableRuntime with the given identifier + * Create a test CompatibleTableRuntime with the given identifier * * @param identifier the table identifier - * @return a DefaultTableRuntime instance + * @return a CompatibleTableRuntime instance */ - private DefaultTableRuntime createDefaultTableRuntime(ServerTableIdentifier identifier) { + private CompatibleTableRuntime createDefaultTableRuntime(ServerTableIdentifier identifier) { // Create table runtime meta TableRuntimeMeta meta = new TableRuntimeMeta(); meta.setTableId(identifier.getId()); @@ -98,9 +98,9 @@ private DefaultTableRuntime createDefaultTableRuntime(ServerTableIdentifier iden // Create table runtime store TableRuntimeStore store = new DefaultTableRuntimeStore( - identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList()); + identifier, meta, CompatibleTableRuntime.REQUIRED_STATES, Collections.emptyList()); - return new DefaultTableRuntime(store); + return new CompatibleTableRuntime(store); } private void cleanUpTableRuntimeData(List tableIds) { @@ -178,7 +178,7 @@ public void testShouldExecuteTaskWithNoPreviousCleanup() { PeriodicTableSchedulerTestBase executor = createTestExecutor(operation); ServerTableIdentifier identifier = createTableIdentifier(1L); - DefaultTableRuntime tableRuntime = createDefaultTableRuntime(identifier); + CompatibleTableRuntime tableRuntime = createDefaultTableRuntime(identifier); boolean shouldExecute = executor.shouldExecuteTaskForTest(tableRuntime, operation); Assert.assertTrue( @@ -203,9 +203,9 @@ public void testShouldNotExecuteTaskWithRecentCleanup() { PeriodicTableSchedulerTestBase executor = createTestExecutor(operation); - // Create DefaultTableRuntime and set recent cleanup time + // Create CompatibleTableRuntime and set recent cleanup time ServerTableIdentifier identifier = createTableIdentifier(1L); - DefaultTableRuntime tableRuntime = createDefaultTableRuntime(identifier); + CompatibleTableRuntime tableRuntime = createDefaultTableRuntime(identifier); // Simulate recent cleanup long recentTime = System.currentTimeMillis() - 10000L; @@ -233,9 +233,9 @@ public void testShouldExecuteTaskWithOldCleanup() { PeriodicTableSchedulerTestBase executor = createTestExecutor(operation); - // Create DefaultTableRuntime and set old cleanup time + // Create CompatibleTableRuntime and set old cleanup time ServerTableIdentifier identifier = createTableIdentifier(1L); - DefaultTableRuntime tableRuntime = createDefaultTableRuntime(identifier); + CompatibleTableRuntime tableRuntime = createDefaultTableRuntime(identifier); // Simulate old cleanup time (30 hours ago) long oldTime = System.currentTimeMillis() - 30 * 60 * 60 * 1000L; @@ -256,7 +256,7 @@ public void testShouldExecuteTaskWithNoneOperation() { PeriodicTableSchedulerTestBase executor = createTestExecutor(CleanupOperation.NONE); ServerTableIdentifier identifier = createTableIdentifier(1L); - DefaultTableRuntime tableRuntime = createDefaultTableRuntime(identifier); + CompatibleTableRuntime tableRuntime = createDefaultTableRuntime(identifier); // Should always execute with NONE operation boolean shouldExecute = executor.shouldExecuteTaskForTest(tableRuntime, CleanupOperation.NONE); diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/AMSTableTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/AMSTableTestBase.java index 5efab334b6..456a5da85e 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/AMSTableTestBase.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/AMSTableTestBase.java @@ -37,6 +37,7 @@ import org.apache.amoro.server.catalog.InternalCatalog; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.table.MixedTable; +import org.apache.amoro.table.TableRuntimeFactory; import org.apache.amoro.utils.CatalogUtil; import org.apache.amoro.utils.ConvertStructUtil; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -69,6 +70,8 @@ public class AMSTableTestBase extends AMSServiceTestBase { private final boolean autoInitTable; private ServerTableIdentifier serverTableIdentifier; + protected final TableRuntimeFactory tableRuntimeFactory; + public AMSTableTestBase(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { this(catalogTestHelper, tableTestHelper, false); } @@ -78,6 +81,7 @@ public AMSTableTestBase( this.catalogTestHelper = catalogTestHelper; this.tableTestHelper = tableTestHelper; this.autoInitTable = autoInitTable; + this.tableRuntimeFactory = new DefaultTableRuntimeFactory(); } @Before diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeHandler.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeHandler.java index 792faebcf7..74ab9837d2 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeHandler.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeHandler.java @@ -41,7 +41,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.mockito.Mockito; import java.util.List; @@ -49,7 +48,6 @@ public class TestDefaultTableRuntimeHandler extends AMSTableTestBase { private DefaultTableService tableService; - private final TableRuntimeFactoryManager runtimeFactoryManager; @Parameterized.Parameters(name = "{0}, {1}") public static Object[] parameters() { @@ -66,16 +64,13 @@ public static Object[] parameters() { public TestDefaultTableRuntimeHandler( CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) { super(catalogTestHelper, tableTestHelper, false); - DefaultTableRuntimeFactory runtimeFactory = new DefaultTableRuntimeFactory(); - runtimeFactoryManager = Mockito.mock(TableRuntimeFactoryManager.class); - Mockito.when(runtimeFactoryManager.installedPlugins()) - .thenReturn(Lists.newArrayList(runtimeFactory)); } @Test public void testInitialize() throws Exception { tableService = - new DefaultTableService(new Configurations(), CATALOG_MANAGER, runtimeFactoryManager); + new DefaultTableService( + new Configurations(), CATALOG_MANAGER, Lists.newArrayList(tableRuntimeFactory)); TestHandler handler = new TestHandler(); tableService.addHandlerChain(handler); tableService.initialize(); @@ -95,7 +90,8 @@ public void testInitialize() throws Exception { // initialize with a history table tableService = - new DefaultTableService(new Configurations(), CATALOG_MANAGER, runtimeFactoryManager); + new DefaultTableService( + new Configurations(), CATALOG_MANAGER, Lists.newArrayList(tableRuntimeFactory)); handler = new TestHandler(); tableService.addHandlerChain(handler); tableService.initialize(); @@ -133,7 +129,8 @@ public void testInitialize() throws Exception { @Test public void testRefreshUpdatesOptimizerGroup() throws Exception { tableService = - new DefaultTableService(new Configurations(), CATALOG_MANAGER, runtimeFactoryManager); + new DefaultTableService( + new Configurations(), CATALOG_MANAGER, Lists.newArrayList(tableRuntimeFactory)); TestHandler handler = new TestHandler(); tableService.addHandlerChain(handler); tableService.initialize(); @@ -144,7 +141,7 @@ public void testRefreshUpdatesOptimizerGroup() throws Exception { createTable(); ServerTableIdentifier tableId = tableManager().listManagedTables().get(0); - DefaultTableRuntime runtime = getDefaultTableRuntime(tableId.getId()); + CompatibleTableRuntime runtime = getDefaultTableRuntime(tableId.getId()); // Verify initial group name is "default" String initialGroup = runtime.getGroupName(); diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeManager.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeManager.java index 17f0e63498..a55dcdfbfd 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeManager.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestDefaultTableRuntimeManager.java @@ -71,7 +71,7 @@ public void testLoadTable() { @Test public void testTableRuntime() { - DefaultTableRuntime tableRuntime = getDefaultTableRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime tableRuntime = getDefaultTableRuntime(serverTableIdentifier().getId()); validateTableRuntime(tableRuntime); } } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestSyncTableOfExternalCatalog.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestSyncTableOfExternalCatalog.java index 9c0ff56237..b15aa1858e 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestSyncTableOfExternalCatalog.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestSyncTableOfExternalCatalog.java @@ -455,7 +455,7 @@ private UnifiedCatalog createNewCatalogTable(String catalogName, String dbName, return externalCatalog; } - @Mock private DefaultTableRuntime tableRuntimeWithException; + @Mock private CompatibleTableRuntime tableRuntimeWithException; @Mock private ServerTableIdentifier tableIdentifierWithException; @Test @@ -535,7 +535,7 @@ public void testDisposeTableRuntimeFailure() { Assert.assertEquals(1, tableRuntimeMetaListForOptimizerGroupAfterAddTable.size()); Assert.assertEquals(1, tableRuntimeMetaListAfterAddTable.size()); - DefaultTableRuntime tableRuntime = getDefaultTableRuntime(tableIdentifier.getId()); + CompatibleTableRuntime tableRuntime = getDefaultTableRuntime(tableIdentifier.getId()); // create a tableIdentifier that throw exceptions when calling the getTableName() method, which // can lead to exceptions when deleting tableIdentifier in disposeTable(). try (AutoCloseable ignored = MockitoAnnotations.openMocks(this)) { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableManagerQuery.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableManagerQuery.java index d5d5ff7416..d05a0014f1 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableManagerQuery.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableManagerQuery.java @@ -86,23 +86,23 @@ public void testGetRuntimes() { // 1.1 add tables with IDLE status // the status will be OptimizingStatus.IDLE default String idle1InGroup1 = "idle1InGroup1"; - DefaultTableRuntime idle1 = + CompatibleTableRuntime idle1 = persistent.newTableRuntime(catalog, db1, idle1InGroup1, TableFormat.ICEBERG, properties); // the status will be OptimizingStatus.IDLE default String idle2InGroup1 = "idle2InGroup1"; - DefaultTableRuntime idle2 = + CompatibleTableRuntime idle2 = persistent.newTableRuntime(catalog, db1, idle2InGroup1, TableFormat.ICEBERG, properties); // 1.2 add tables with PENDING status String pending1InGroup1 = "pending1InGroup1"; - DefaultTableRuntime pending1 = + CompatibleTableRuntime pending1 = persistent.newTableRuntime(catalog, db1, pending1InGroup1, TableFormat.ICEBERG, properties); // update status pending1.setPendingInput(new AbstractOptimizingEvaluator.PendingInput()); String pending2InGroup1 = "pending2InGroup1"; - DefaultTableRuntime pending2 = + CompatibleTableRuntime pending2 = persistent.newTableRuntime(catalog, db1, pending2InGroup1, TableFormat.ICEBERG, properties); // update status pending2.setPendingInput(new AbstractOptimizingEvaluator.PendingInput()); @@ -110,31 +110,31 @@ public void testGetRuntimes() { // 1.3 add tables with PLANNING status String db2 = "db2"; String plan1InGroup1 = "plan1InGroup1"; - DefaultTableRuntime plan1 = + CompatibleTableRuntime plan1 = persistent.newTableRuntime(catalog, db2, plan1InGroup1, TableFormat.ICEBERG, properties); plan1.beginPlanning(); String plan2InGroup1 = "plan2InGroup1"; - DefaultTableRuntime plan2 = + CompatibleTableRuntime plan2 = persistent.newTableRuntime(catalog, db2, plan2InGroup1, TableFormat.ICEBERG, properties); plan2.beginPlanning(); // 1.4 add tables with COMMITTING status String committing1InGroup1 = "committing1InGroup1"; - DefaultTableRuntime committing1 = + CompatibleTableRuntime committing1 = persistent.newTableRuntime( catalog, db2, committing1InGroup1, TableFormat.ICEBERG, properties); committing1.beginCommitting(); String commiting2InGroup1 = "committing2InGroup1"; - DefaultTableRuntime committing2 = + CompatibleTableRuntime committing2 = persistent.newTableRuntime( catalog, db2, commiting2InGroup1, TableFormat.ICEBERG, properties); committing2.beginCommitting(); // 1.5 add tables with MINOR_OPTIMIZING status String minor1InGroup1 = "minor1InGroup1"; - DefaultTableRuntime minor1 = + CompatibleTableRuntime minor1 = persistent.newTableRuntime(catalog, db2, minor1InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process = mock(OptimizingProcess.class); doReturn(1L).when(process).getProcessId(); @@ -142,7 +142,7 @@ public void testGetRuntimes() { minor1.beginProcess(process); String minor2InGroup1 = "minor2InGroup1"; - DefaultTableRuntime minor2 = + CompatibleTableRuntime minor2 = persistent.newTableRuntime(catalog, db2, minor2InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process2 = mock(OptimizingProcess.class); doReturn(2L).when(process2).getProcessId(); @@ -151,7 +151,7 @@ public void testGetRuntimes() { // 1.6 add tables with MAJOR_OPTIMIZING status String major1InGroup1 = "major1InGroup1"; - DefaultTableRuntime major1 = + CompatibleTableRuntime major1 = persistent.newTableRuntime(catalog, db1, major1InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process3 = mock(OptimizingProcess.class); doReturn(3L).when(process3).getProcessId(); @@ -159,7 +159,7 @@ public void testGetRuntimes() { major1.beginProcess(process3); String major2InGroup1 = "major2InGroup1"; - DefaultTableRuntime major2 = + CompatibleTableRuntime major2 = persistent.newTableRuntime(catalog, db1, major2InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process4 = mock(OptimizingProcess.class); doReturn(4L).when(process4).getProcessId(); @@ -168,7 +168,7 @@ public void testGetRuntimes() { // 1.7 add tables with FULL_OPTIMIZING status String full1InGroup1 = "full1InGroup1"; - DefaultTableRuntime full1 = + CompatibleTableRuntime full1 = persistent.newTableRuntime(catalog, db1, full1InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process5 = mock(OptimizingProcess.class); doReturn(5L).when(process5).getProcessId(); @@ -176,7 +176,7 @@ public void testGetRuntimes() { full1.beginProcess(process5); String full2InGroup1 = "full2InGroup1"; - DefaultTableRuntime full2 = + CompatibleTableRuntime full2 = persistent.newTableRuntime(catalog, db1, full2InGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process6 = mock(OptimizingProcess.class); doReturn(6L).when(process6).getProcessId(); @@ -188,7 +188,7 @@ public void testGetRuntimes() { String opGroup2 = "opGroup2-other"; properties.put(TableProperties.SELF_OPTIMIZING_GROUP, opGroup2); String minor1InOtherGroup1 = "minor1-InOtherGroup"; - DefaultTableRuntime minor1Other = + CompatibleTableRuntime minor1Other = persistent.newTableRuntime( catalog, db1, minor1InOtherGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process7 = mock(OptimizingProcess.class); @@ -197,7 +197,7 @@ public void testGetRuntimes() { minor1Other.beginProcess(process7); String minor2InOtherGroup1 = "minor2-InOtherGroup"; - DefaultTableRuntime minor2Other = + CompatibleTableRuntime minor2Other = persistent.newTableRuntime( catalog, db1, minor2InOtherGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process8 = mock(OptimizingProcess.class); @@ -206,7 +206,7 @@ public void testGetRuntimes() { minor2Other.beginProcess(process8); String minor3InOtherGroup1 = "minor3-InOtherGroup"; - DefaultTableRuntime minor3Other = + CompatibleTableRuntime minor3Other = persistent.newTableRuntime( catalog, db1, minor3InOtherGroup1, TableFormat.ICEBERG, properties); OptimizingProcess process9 = mock(OptimizingProcess.class); @@ -300,7 +300,7 @@ public ServerTableIdentifier newIdentifier( return identifier; } - public DefaultTableRuntime newTableRuntime( + public CompatibleTableRuntime newTableRuntime( String catalog, String database, String tableName, @@ -317,8 +317,8 @@ public DefaultTableRuntime newTableRuntime( doAs(TableRuntimeMapper.class, mapper -> mapper.insertRuntime(meta)); DefaultTableRuntimeStore store = new DefaultTableRuntimeStore( - identifier, meta, DefaultTableRuntime.REQUIRED_STATES, Collections.emptyList()); - return new DefaultTableRuntime(store); + identifier, meta, CompatibleTableRuntime.REQUIRED_STATES, Collections.emptyList()); + return new CompatibleTableRuntime(store); } } } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java index acf3a73cec..13ef57921e 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java @@ -105,7 +105,7 @@ private void initTableWithFiles() { .asUnkeyedTable(); appendData(table); appendPosDelete(table); - DefaultTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); + CompatibleTableRuntime runtime = getDefaultTableRuntime(serverTableIdentifier().getId()); runtime.refresh(tableService().loadTable(serverTableIdentifier())); } diff --git a/amoro-common/src/main/java/org/apache/amoro/Action.java b/amoro-common/src/main/java/org/apache/amoro/Action.java index 42671a6e1d..1d5f81b3bf 100644 --- a/amoro-common/src/main/java/org/apache/amoro/Action.java +++ b/amoro-common/src/main/java/org/apache/amoro/Action.java @@ -20,38 +20,33 @@ import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; -import java.util.Arrays; +import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; public final class Action { private static final int MAX_NAME_LENGTH = 16; - - /** supported table formats of this action */ - private final TableFormat[] formats; + private static final Map registeredActions = new ConcurrentHashMap<>(); private final String name; - /** - * the weight number of this action, the bigger the weight number, the higher positions of - * schedulers or front pages - */ - private final int weight; - public Action(TableFormat[] formats, int weight, String name) { - Preconditions.checkArgument( - name.length() <= MAX_NAME_LENGTH, - "Action name length should be less than " + MAX_NAME_LENGTH); - this.formats = formats; - this.name = name; - this.weight = weight; + public static Action register(String name) { + final String regularName = name.trim().toUpperCase(Locale.ROOT); + return registeredActions.computeIfAbsent(regularName, s -> new Action(regularName)); } - public int getWeight() { - return weight; + public static Action valueOf(String name) { + final String regularName = name.trim().toUpperCase(Locale.ROOT); + return registeredActions.get(regularName); } - public TableFormat[] supportedFormats() { - return formats; + private Action(String name) { + Preconditions.checkArgument( + name.length() <= MAX_NAME_LENGTH, + "Action name length should be less than " + MAX_NAME_LENGTH); + this.name = name; } public String getName() { @@ -67,13 +62,11 @@ public boolean equals(Object o) { return false; } Action action = (Action) o; - return Objects.equals(name, action.name) && Arrays.equals(formats, action.formats); + return Objects.equals(name, action.name); } @Override public int hashCode() { - int result = Objects.hash(name); - result = 31 * result + Arrays.hashCode(formats); - return result; + return Objects.hash(name); } } diff --git a/amoro-common/src/main/java/org/apache/amoro/IcebergActions.java b/amoro-common/src/main/java/org/apache/amoro/IcebergActions.java index 76f470d98c..c75c5ac8d0 100644 --- a/amoro-common/src/main/java/org/apache/amoro/IcebergActions.java +++ b/amoro-common/src/main/java/org/apache/amoro/IcebergActions.java @@ -23,9 +23,9 @@ public class IcebergActions { private static final TableFormat[] DEFAULT_FORMATS = new TableFormat[] {TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.MIXED_HIVE}; - public static final Action SYSTEM = new Action(DEFAULT_FORMATS, 0, "system"); - public static final Action REWRITE = new Action(DEFAULT_FORMATS, 10, "rewrite"); - public static final Action DELETE_ORPHANS = new Action(DEFAULT_FORMATS, 2, "delete-orphans"); - public static final Action SYNC_HIVE = new Action(DEFAULT_FORMATS, 3, "sync-hive"); - public static final Action EXPIRE_DATA = new Action(DEFAULT_FORMATS, 1, "expire-data"); + public static final Action SYSTEM = Action.register("system"); + public static final Action REWRITE = Action.register("rewrite"); + public static final Action DELETE_ORPHANS = Action.register("delete-orphans"); + public static final Action SYNC_HIVE = Action.register("sync-hive"); + public static final Action EXPIRE_DATA = Action.register("expire-data"); } diff --git a/amoro-common/src/main/java/org/apache/amoro/TableRuntime.java b/amoro-common/src/main/java/org/apache/amoro/TableRuntime.java index 31ea74f1fe..9fa9a29e47 100644 --- a/amoro-common/src/main/java/org/apache/amoro/TableRuntime.java +++ b/amoro-common/src/main/java/org/apache/amoro/TableRuntime.java @@ -48,6 +48,10 @@ public interface TableRuntime { */ List getProcessStates(Action action); + void registerProcess(TableProcessStore processStore); + + void removeProcess(TableProcessStore processStore); + /** Get the group name of the table runtime. */ String getGroupName(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinator.java b/amoro-common/src/main/java/org/apache/amoro/process/ActionCoordinator.java similarity index 70% rename from amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinator.java rename to amoro-common/src/main/java/org/apache/amoro/process/ActionCoordinator.java index 5e55154cbd..18af6387ec 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ActionCoordinator.java +++ b/amoro-common/src/main/java/org/apache/amoro/process/ActionCoordinator.java @@ -16,25 +16,19 @@ * limitations under the License. */ -package org.apache.amoro.server.process; +package org.apache.amoro.process; import org.apache.amoro.Action; -import org.apache.amoro.ActivePlugin; import org.apache.amoro.TableFormat; import org.apache.amoro.TableRuntime; -import org.apache.amoro.process.TableProcess; -import org.apache.amoro.process.TableProcessStore; -import org.apache.amoro.server.utils.SnowflakeIdGenerator; + +import java.util.Optional; /** * Coordinator for a specific {@link org.apache.amoro.Action} to manage table processes. Provides * scheduling parameters and lifecycle hooks to create/recover/cancel/retry table processes. */ -public interface ActionCoordinator extends ActivePlugin { - - String PROPERTY_PARALLELISM = "parallelism"; - - SnowflakeIdGenerator SNOWFLAKE_ID_GENERATOR = new SnowflakeIdGenerator(); +public interface ActionCoordinator { /** * Check whether the given table format is supported by this coordinator. @@ -58,13 +52,6 @@ public interface ActionCoordinator extends ActivePlugin { */ Action action(); - /** - * Get the execution engine name used by this coordinator. - * - * @return execution engine name - */ - String executionEngine(); - /** * Calculate the next executing time for the given table runtime. * @@ -94,7 +81,7 @@ public interface ActionCoordinator extends ActivePlugin { * @param tableRuntime table runtime * @return a new table process */ - TableProcess createTableProcess(TableRuntime tableRuntime); + Optional trigger(TableRuntime tableRuntime); /** * Recover a {@link TableProcess} from persisted store. @@ -104,21 +91,4 @@ public interface ActionCoordinator extends ActivePlugin { * @return recovered table process */ TableProcess recoverTableProcess(TableRuntime tableRuntime, TableProcessStore processStore); - - /** - * Prepare a {@link TableProcess} for cancellation. - * - * @param tableRuntime table runtime - * @param process table process to cancel - * @return the process instance to be canceled - */ - TableProcess cancelTableProcess(TableRuntime tableRuntime, TableProcess process); - - /** - * Prepare a {@link TableProcess} for retrying. - * - * @param process table process to retry - * @return the process instance to be retried - */ - TableProcess retryTableProcess(TableProcess process); } diff --git a/amoro-common/src/main/java/org/apache/amoro/process/AmoroProcess.java b/amoro-common/src/main/java/org/apache/amoro/process/AmoroProcess.java index cf0ffd50bf..6ff5b6cfb6 100644 --- a/amoro-common/src/main/java/org/apache/amoro/process/AmoroProcess.java +++ b/amoro-common/src/main/java/org/apache/amoro/process/AmoroProcess.java @@ -47,7 +47,7 @@ public interface AmoroProcess { SimpleFuture getCompleteFuture(); /** - * Get {@link ProcessState} of the process + * Get {@link TableProcessStore} of the process * * @return the state of the process */ diff --git a/amoro-common/src/main/java/org/apache/amoro/process/OptimizingState.java b/amoro-common/src/main/java/org/apache/amoro/process/OptimizingState.java deleted file mode 100644 index c4a900dcac..0000000000 --- a/amoro-common/src/main/java/org/apache/amoro/process/OptimizingState.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.process; - -import org.apache.amoro.Action; -import org.apache.amoro.ServerTableIdentifier; -import org.apache.amoro.StateField; - -/** The state of the optimizing process. */ -public abstract class OptimizingState extends TableProcessState { - - @StateField private volatile long targetSnapshotId; - @StateField private volatile long watermark; - @StateField private volatile ProcessStage stage; - @StateField private volatile long currentStageStartTime; - - public OptimizingState(Action action, ServerTableIdentifier tableIdentifier) { - super(action, tableIdentifier); - } - - public OptimizingState(long id, Action action, ServerTableIdentifier tableIdentifier) { - super(id, action, tableIdentifier); - } - - protected void setStage(ProcessStage stage) { - this.stage = stage; - this.currentStageStartTime = System.currentTimeMillis(); - } - - protected void setStage(ProcessStage stage, long stageStartTime) { - this.stage = stage; - this.currentStageStartTime = stageStartTime; - } - - protected void setTargetSnapshotId(long targetSnapshotId) { - this.targetSnapshotId = targetSnapshotId; - } - - protected void setWatermark(long watermark) { - this.watermark = watermark; - } - - public long getWatermark() { - return watermark; - } - - @Override - public ProcessStage getStage() { - return stage; - } - - public long getTargetSnapshotId() { - return targetSnapshotId; - } - - public long getCurrentStageStartTime() { - return currentStageStartTime; - } - - @Override - public String getName() { - return stage.getDesc(); - } -} diff --git a/amoro-common/src/main/java/org/apache/amoro/process/ProcessFactory.java b/amoro-common/src/main/java/org/apache/amoro/process/ProcessFactory.java index a2df0a99f3..6e325d6934 100644 --- a/amoro-common/src/main/java/org/apache/amoro/process/ProcessFactory.java +++ b/amoro-common/src/main/java/org/apache/amoro/process/ProcessFactory.java @@ -19,23 +19,42 @@ package org.apache.amoro.process; import org.apache.amoro.Action; +import org.apache.amoro.ActivePlugin; +import org.apache.amoro.TableFormat; import org.apache.amoro.TableRuntime; +import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; +import org.apache.amoro.table.StateKey; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; /** * A factory to create a process. Normally, There will be default ProcessFactories for each action * and used by default scheduler. Meanwhile, user could extend external ProcessFactory to run jobs * on external resources like Yarn. */ -public interface ProcessFactory { +public interface ProcessFactory extends ActivePlugin { + + default List> requiredStates() { + return Lists.newArrayList(); + } + + /** Get supported actions for each table format. */ + Map> supportedActions(); + + /** How to trigger a process for the action. */ + ProcessTriggerStrategy triggerStrategy(TableFormat format, Action action); /** - * Create a process for the action. + * Try trigger a process for the action. * * @param tableRuntime table runtime * @param action action type * @return target process which has not been submitted yet. */ - AmoroProcess create(TableRuntime tableRuntime, Action action); + Optional trigger(TableRuntime tableRuntime, Action action); /** * Recover a process for the action from a state. @@ -44,5 +63,5 @@ public interface ProcessFactory { * @param state state of the process * @return target process which has not been submitted yet. */ - AmoroProcess recover(TableRuntime tableRuntime, TableProcessState state); + TableProcess recover(TableRuntime tableRuntime, TableProcessStore store); } diff --git a/amoro-common/src/main/java/org/apache/amoro/process/ProcessState.java b/amoro-common/src/main/java/org/apache/amoro/process/ProcessState.java deleted file mode 100644 index 91f76fcaef..0000000000 --- a/amoro-common/src/main/java/org/apache/amoro/process/ProcessState.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.process; - -import org.apache.amoro.Action; - -import java.util.Map; - -/** - * ProcessState contains information in any {@link AmoroProcess} which must be persistent and {@link - * ProcessFactory} will use to recover {@link AmoroProcess}. - */ -public interface ProcessState { - - /** @return unique identifier of the process. */ - long getId(); - - /** - * @return the name of the state. If multiple stages are involved, it should be the name of the - * current stage. - */ - String getName(); - - /** @return start time of the process. */ - long getStartTime(); - - /** @return the action of the process. */ - Action getAction(); - - /** @return the status of the process. */ - ProcessStatus getStatus(); - - /** - * Get the string encoded summary of the process, this could be a simple description or a POJO - * encoded by JSON - * - * @return the summary of the process - */ - Map getSummary(); - - /** @return the reason of process failure, null if the process has not failed yet. */ - String getFailedReason(); - - /** - * Total millisecond running time of all tasks in the process. - * - * @return actual quota runtime of the process. - */ - long getQuotaRuntime(); - - /** - * Quota value is calculated by the total millisecond running time of all tasks in the process - * divided by the total millisecond from the start time to the current time. It is used to - * evaluate the actual runtime concurrence of the process. - * - * @return the quota value of the process. - */ - default double getQuotaValue() { - return (double) getQuotaRuntime() / (System.currentTimeMillis() - getStartTime()); - } -} diff --git a/amoro-common/src/main/java/org/apache/amoro/process/ProcessTriggerStrategy.java b/amoro-common/src/main/java/org/apache/amoro/process/ProcessTriggerStrategy.java new file mode 100644 index 0000000000..2f54a4ed20 --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/process/ProcessTriggerStrategy.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.process; + +import java.time.Duration; + +/** Process trigger strategy. */ +public final class ProcessTriggerStrategy { + + private final Duration triggerInterval; + + private final boolean triggerOnNewSnapshot; + + private final int triggerParallelism; + + public ProcessTriggerStrategy( + Duration triggerInterval, boolean triggerOnNewSnapshot, int triggerParallelism) { + this.triggerInterval = triggerInterval; + this.triggerOnNewSnapshot = triggerOnNewSnapshot; + this.triggerParallelism = triggerParallelism; + } + + public static ProcessTriggerStrategy triggerAtFixRate(Duration triggerInterval) { + return new ProcessTriggerStrategy(triggerInterval, false, 1); + } + + public Duration getTriggerInterval() { + return triggerInterval; + } + + public boolean isTriggerOnNewSnapshot() { + return triggerOnNewSnapshot; + } + + public int getTriggerParallelism() { + return triggerParallelism; + } +} diff --git a/amoro-common/src/main/java/org/apache/amoro/process/TableProcessState.java b/amoro-common/src/main/java/org/apache/amoro/process/TableProcessState.java deleted file mode 100644 index c4e800ed1c..0000000000 --- a/amoro-common/src/main/java/org/apache/amoro/process/TableProcessState.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.process; - -import org.apache.amoro.Action; -import org.apache.amoro.ServerTableIdentifier; -import org.apache.amoro.StateField; - -import java.util.Map; - -/** A common state of a table process. */ -public class TableProcessState implements ProcessState { - - @StateField private volatile long id; - @StateField private volatile String externalProcessIdentifier; - private final Action action; - private final ServerTableIdentifier tableIdentifier; - private String executionEngine; - @StateField private int retryNumber = 0; - @StateField private long startTime = -1L; - @StateField private long endTime = -1L; - @StateField private ProcessStatus status = ProcessStatus.PENDING; - @StateField private volatile String failedReason; - private volatile Map processParameters; - private volatile Map summary; - - public TableProcessState(Action action, ServerTableIdentifier tableIdentifier) { - this.action = action; - this.tableIdentifier = tableIdentifier; - } - - public TableProcessState(long id, Action action, ServerTableIdentifier tableIdentifier) { - this.id = id; - this.action = action; - this.tableIdentifier = tableIdentifier; - } - - public TableProcessState( - long id, Action action, ServerTableIdentifier tableIdentifier, String executionEngine) { - this.id = id; - this.action = action; - this.tableIdentifier = tableIdentifier; - this.executionEngine = executionEngine; - } - - @Override - public long getId() { - return id; - } - - public String getExternalProcessIdentifier() { - return externalProcessIdentifier; - } - - public String getName() { - return action.getName(); - } - - public Action getAction() { - return action; - } - - public long getStartTime() { - return startTime; - } - - public long getEndTime() { - return endTime; - } - - public ProcessStatus getStatus() { - return status; - } - - public String getExecutionEngine() { - return executionEngine; - } - - @Override - public Map getSummary() { - return summary; - } - - @Override - public long getQuotaRuntime() { - return getDuration(); - } - - @Override - public double getQuotaValue() { - return 1; - } - - public long getDuration() { - return endTime > 0 ? endTime - startTime : System.currentTimeMillis() - startTime; - } - - public ServerTableIdentifier getTableIdentifier() { - return tableIdentifier; - } - - public void setExternalProcessIdentifier(String externalProcessIdentifier) { - this.externalProcessIdentifier = externalProcessIdentifier; - } - - public void setExecutionEngine(String executionEngine) { - this.executionEngine = executionEngine; - } - - protected void setSummary(Map summary) { - this.summary = summary; - } - - protected void setStartTime(long startTime) { - this.startTime = startTime; - } - - public void setStatus(ProcessStatus status) { - if (status == ProcessStatus.SUCCESS - || status == ProcessStatus.FAILED - || status == ProcessStatus.KILLED) { - endTime = System.currentTimeMillis(); - } else if (this.status != ProcessStatus.SUBMITTED && status == ProcessStatus.SUBMITTED) { - endTime = -1L; - failedReason = null; - summary = null; - } - this.status = status; - } - - public String getFailedReason() { - return failedReason; - } - - public ProcessStage getStage() { - return status.toStage(); - } - - protected void setId(long processId) { - this.id = processId; - } - - public Map getProcessParameters() { - return processParameters; - } - - public void setProcessParameters(Map processParameters) { - this.processParameters = processParameters; - } - - public void setSubmitted(String externalProcessIdentifier) { - this.status = ProcessStatus.SUBMITTED; - setExternalProcessIdentifier(externalProcessIdentifier); - this.startTime = System.currentTimeMillis(); - } - - public void setSubmitted() { - this.status = ProcessStatus.SUBMITTED; - this.startTime = System.currentTimeMillis(); - } - - public void setRunning() { - this.status = ProcessStatus.RUNNING; - } - - public void setCanceling() { - this.status = ProcessStatus.CANCELING; - } - - public void addRetryNumber() { - this.retryNumber += 1; - this.status = ProcessStatus.PENDING; - this.externalProcessIdentifier = ""; - this.failedReason = null; - } - - public void resetRetryNumber() { - this.retryNumber = 0; - } - - public void setCanceled() { - this.status = ProcessStatus.CANCELED; - } - - public void setCompleted() { - this.status = ProcessStatus.SUCCESS; - this.endTime = System.currentTimeMillis(); - } - - public void setKilled() { - this.status = ProcessStatus.KILLED; - this.endTime = System.currentTimeMillis(); - } - - public void setCompleted(String failedReason) { - this.status = ProcessStatus.FAILED; - this.failedReason = failedReason; - this.endTime = System.currentTimeMillis(); - } - - public int getRetryNumber() { - return retryNumber; - } - - public void setRetryNumber(int retryNumber) { - this.retryNumber = retryNumber; - } -} diff --git a/amoro-common/src/main/java/org/apache/amoro/table/TableRuntimeFactory.java b/amoro-common/src/main/java/org/apache/amoro/table/TableRuntimeFactory.java index 7654af7104..90e26bd4b6 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/TableRuntimeFactory.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/TableRuntimeFactory.java @@ -21,6 +21,8 @@ import org.apache.amoro.ActivePlugin; import org.apache.amoro.ServerTableIdentifier; import org.apache.amoro.TableRuntime; +import org.apache.amoro.process.ActionCoordinator; +import org.apache.amoro.process.ProcessFactory; import java.util.List; import java.util.Map; @@ -29,6 +31,10 @@ /** Table runtime factory. */ public interface TableRuntimeFactory extends ActivePlugin { + List supportedCoordinators(); + + void initialize(List factories); + Optional accept( ServerTableIdentifier tableIdentifier, Map tableProperties);