diff --git a/alter/table/replace_partition/common.py b/alter/table/replace_partition/common.py
index 59bad00a3..7bb86b8cb 100644
--- a/alter/table/replace_partition/common.py
+++ b/alter/table/replace_partition/common.py
@@ -9,6 +9,26 @@
from helpers.tables import create_table_partitioned_by_column, create_table, Column
+@TestStep(Given)
+def create_partitions_for_collapsing_merge_tree(
+ self,
+ table_name,
+ number_of_values=3,
+ number_of_partitions=5,
+ number_of_parts=1,
+ node=None,
+):
+ """Insert random UInt64 values into a column and create multiple partitions based on the value of number_of_partitions."""
+ if node is None:
+ node = self.context.node
+
+ with By("Inserting random values into a column with uint64 datatype"):
+ for i in range(1, number_of_partitions + 1):
+ for parts in range(1, number_of_parts + 1):
+ node.query(
+ f"INSERT INTO {table_name} (p, i) SELECT {random.choice([-1, 1])}, rand64() FROM numbers({number_of_values})"
+ )
+
@TestStep(Given)
def create_partitions_with_random_uint64(
self,
diff --git a/engines/tests/replacing_merge_tree/replacing_merge_tree.py b/engines/tests/replacing_merge_tree/replacing_merge_tree.py
index 495dcc10e..cc4f3929e 100644
--- a/engines/tests/replacing_merge_tree/replacing_merge_tree.py
+++ b/engines/tests/replacing_merge_tree/replacing_merge_tree.py
@@ -533,7 +533,7 @@ def incorrect_data_insert_with_disabled_optimize_on_insert(self, node=None):
):
node.query(f"SELECT * FROM {name} FORMAT JSONEachRow;")
- with And("I optimize table"):
+ with By("optimizing a table"):
node.query(
f"OPTIMIZE TABLE {name} FINAL;",
message="DB::Exception:",
diff --git a/helpers/cluster.py b/helpers/cluster.py
index 434f68bdb..b9133ce3d 100755
--- a/helpers/cluster.py
+++ b/helpers/cluster.py
@@ -1012,6 +1012,7 @@ def query(
ignore_exception=False,
step=By,
settings=None,
+ inline_settings=None,
retry_count=5,
messages_to_retry=None,
retry_delay=5,
@@ -1035,6 +1036,7 @@ def query(
:param no_check: disable exitcode and message checks, default: False
:param step: wrapping step class, default: By
:param settings: list of settings to be used for the query in the form [(name, value),...], default: None
+ :param inline_settings: list of inline settings to be used for the query in the form [(name, value),...], default: None
:param retry_count: number of retries, default: 5
:param messages_to_retry: list of messages in the query output for
which retry should be triggered, default: MESSAGES_TO_RETRY
@@ -1057,6 +1059,7 @@ def query(
retry_count = max(0, int(retry_count))
retry_delay = max(0, float(retry_delay))
settings = list(settings or [])
+ inline_settings = list(inline_settings or [])
query_settings = list(settings)
if raise_on_exception:
@@ -1077,6 +1080,13 @@ def query(
if query_id is not None:
query_settings += [("query_id", f"{query_id}")]
+ if inline_settings:
+ sql = (
+ "; ".join([f"SET {name} = {value}" for name, value in inline_settings])
+ + "; "
+ + sql
+ )
+
client = "clickhouse client -n"
if secure:
client += (
diff --git a/helpers/create.py b/helpers/create.py
index 8472a9019..16b19c8f3 100644
--- a/helpers/create.py
+++ b/helpers/create.py
@@ -1,6 +1,9 @@
from testflows.core import *
-from alter.table.replace_partition.common import create_partitions_with_random_uint64
+from alter.table.replace_partition.common import (
+ create_partitions_with_random_uint64,
+ create_partitions_for_collapsing_merge_tree,
+)
@TestStep(Given)
@@ -21,6 +24,8 @@ def create_table(
comment=None,
settings=None,
partition_by=None,
+ stop_merges=False,
+ query_settings=None,
):
"""
Generates a query to create a table in ClickHouse.
@@ -105,13 +110,23 @@ def create_table(
if comment:
query += f" COMMENT '{comment}'"
+ if query_settings:
+ query += f" SETTINGS {query_settings}"
+
query += ";"
+ if stop_merges:
+ query += f" SYSTEM STOP MERGES {table_name};"
+
node.query(query)
yield
+
finally:
with Finally(f"drop the table {table_name}"):
- node.query(f"DROP TABLE IF EXISTS {table_name}")
+ query = f"DROP TABLE IF EXISTS {table_name}"
+ if cluster:
+ query += f" ON CLUSTER {cluster}"
+ node.query(query)
return query
@@ -127,6 +142,9 @@ def create_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = None,
+ cluster: str = None,
+ stop_merges: bool = False,
+ query_settings: str = None,
):
"""Create a table with the MergeTree engine."""
create_table(
@@ -139,6 +157,9 @@ def create_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
+ query_settings=query_settings,
)
@@ -153,6 +174,8 @@ def create_replacing_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = None,
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the ReplacingMergeTree engine."""
create_table(
@@ -165,6 +188,8 @@ def create_replacing_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -179,6 +204,8 @@ def create_summing_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = None,
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the SummingMergeTree engine."""
create_table(
@@ -191,6 +218,8 @@ def create_summing_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -205,6 +234,8 @@ def create_aggregating_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = None,
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the AggregatingMergeTree engine."""
create_table(
@@ -217,6 +248,8 @@ def create_aggregating_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -232,6 +265,8 @@ def create_collapsing_merge_tree_table(
order_by: str = "tuple()",
partition_by: str = None,
sign: str = "Sign",
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the CollapsingMergeTree engine.
@@ -248,6 +283,8 @@ def create_collapsing_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -264,6 +301,8 @@ def create_versioned_collapsing_merge_tree_table(
partition_by: str = None,
sign: str = "Sign",
version: str = "Version",
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the VersionedCollapsingMergeTree engine.
@@ -281,6 +320,8 @@ def create_versioned_collapsing_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -296,6 +337,8 @@ def create_graphite_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = None,
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the GraphiteMergeTree engine.
@@ -312,6 +355,8 @@ def create_graphite_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@@ -326,6 +371,8 @@ def create_replicated_merge_tree_table(
primary_key=None,
order_by: str = "tuple()",
partition_by: str = "p",
+ cluster: str = None,
+ stop_merges: bool = False,
):
"""Create a table with the MergeTree engine."""
if columns is None:
@@ -345,84 +392,200 @@ def create_replicated_merge_tree_table(
db=db,
comment=comment,
partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
@TestStep(Given)
-def partitioned_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
+ number_of_values=3,
+ query_settings=None,
+):
"""Create a MergeTree table partitioned by a specific column."""
with By(f"creating a partitioned {table_name} table with a MergeTree engine"):
create_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
+ query_settings=query_settings,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ number_of_values=number_of_values,
+ )
+
+ return table_name
@TestStep(Given)
def partitioned_replicated_merge_tree_table(
- self, table_name, partition_by, columns=None
+ self,
+ table_name,
+ partition_by,
+ columns=None,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
):
"""Create a ReplicatedMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a ReplicatedMergeTree engine"
):
create_replicated_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
-def partitioned_replacing_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_replacing_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
+):
"""Create a ReplacingMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a ReplacingMergeTree engine"
):
create_replacing_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
-def partitioned_summing_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_summing_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
+):
"""Create a SummingMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a SummingMergeTree engine"
):
create_aggregating_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
-def partitioned_collapsing_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_collapsing_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=1,
+ number_of_parts=1,
+):
"""Create a CollapsingMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a CollapsingMergeTree engine"
):
create_collapsing_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by, sign="p"
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ sign="p",
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(
- table_name=table_name, number_of_partitions=1
- )
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_for_collapsing_merge_tree(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
def partitioned_versioned_collapsing_merge_tree_table(
- self, table_name, partition_by, columns
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=1,
+ number_of_parts=1,
):
"""Create a VersionedCollapsingMergeTree table partitioned by a specific column."""
with By(
@@ -434,30 +597,68 @@ def partitioned_versioned_collapsing_merge_tree_table(
partition_by=partition_by,
sign="p",
version="i",
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(
- table_name=table_name, number_of_partitions=1
- )
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
-def partitioned_aggregating_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_aggregating_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
+):
"""Create a AggregatingMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a AggregatingMergeTree engine"
):
create_summing_merge_tree_table(
- table_name=table_name, columns=columns, partition_by=partition_by
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
@TestStep(Given)
-def partitioned_graphite_merge_tree_table(self, table_name, partition_by, columns):
+def partitioned_graphite_merge_tree_table(
+ self,
+ table_name,
+ partition_by,
+ columns,
+ cluster=None,
+ stop_merges=False,
+ populate=True,
+ number_of_partitions=5,
+ number_of_parts=1,
+):
"""Create a GraphiteMergeTree table partitioned by a specific column."""
with By(
f"creating a partitioned {table_name} table with a GraphiteMergeTree engine"
@@ -467,7 +668,16 @@ def partitioned_graphite_merge_tree_table(self, table_name, partition_by, column
columns=columns,
partition_by=partition_by,
config="graphite_rollup_example",
+ cluster=cluster,
+ stop_merges=stop_merges,
)
- with And("populating it with the data needed to create multiple partitions"):
- create_partitions_with_random_uint64(table_name=table_name)
+ if populate:
+ with And("populating it with the data needed to create multiple partitions"):
+ create_partitions_with_random_uint64(
+ table_name=table_name,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+ return table_name
\ No newline at end of file
diff --git a/helpers/queries.py b/helpers/queries.py
index de142476b..bed43009b 100644
--- a/helpers/queries.py
+++ b/helpers/queries.py
@@ -16,6 +16,30 @@
# The extra [0] could be avoided with TSV format, but that does not guarantee valid JSON.
+@TestStep(Given)
+def get_cluster_nodes(self, cluster, node=None):
+ """Get all nodes in a cluster."""
+
+ if node is None:
+ node = self.context.node
+
+ result = node.query(
+ f"SELECT host_name FROM system.clusters WHERE cluster = '{cluster}'", exitcode=0
+ )
+
+ nodes = [line.strip() for line in result.output.splitlines() if line.strip()]
+ return nodes
+
+
+@TestStep(When)
+def select_all_ordered(self, table_name, node, order_by="p, i"):
+ """Select all data from a table ordered by partition and index columns."""
+
+ return node.query(
+ f"SELECT * FROM {table_name} ORDER BY {order_by}", exitcode=0
+ ).output.splitlines()
+
+
@TestStep(When)
def sync_replica(
self, node: ClickHouseNode, table_name: str, raise_on_timeout=False, **kwargs
@@ -124,3 +148,12 @@ def get_column_string(self, node: ClickHouseNode, table_name: str, timeout=30) -
timeout=timeout,
)
return ",".join([l.strip() for l in r.output.splitlines()])
+
+
+@TestStep(When)
+def drop_column(self, node, table_name, column_name):
+ """Drop a column from a table."""
+
+ node.query(
+ f"ALTER TABLE {table_name} DROP COLUMN {column_name}", exitcode=0, steps=True
+ )
\ No newline at end of file
diff --git a/helpers/tables.py b/helpers/tables.py
index 1f209ed92..54fe10e1d 100644
--- a/helpers/tables.py
+++ b/helpers/tables.py
@@ -343,9 +343,10 @@ def generate_all_map_column_types():
class Table:
- def __init__(self, name, columns, engine):
+ def __init__(self, name, columns, engine, partition_by=None):
self.name = name
self.columns = columns
+ self.partition_by = partition_by
self.engine = engine
def insert_test_data(
@@ -422,6 +423,7 @@ def create_table(
node=None,
cluster=None,
order_by_all_columns=False,
+ stop_merges=False,
):
"""Create a table with specified name and engine."""
if settings is None:
@@ -479,12 +481,19 @@ def create_table(
if query_settings is not None:
query += f"\nSETTINGS {query_settings}"
+ query += ";"
+
+ if stop_merges:
+ query += f" SYSTEM STOP MERGES {name};"
+
node.query(
query,
settings=settings,
)
- yield Table(name, columns, engine)
+ yield Table(
+ name=name, columns=columns, partition_by=partition_by, engine=engine
+ )
finally:
with Finally(f"drop the table {name}"):
@@ -547,7 +556,7 @@ def create_temporary_table(
settings=settings,
)
- yield Table(name, columns, engine)
+ yield Table(name, columns, engine, partition_by=partition_by)
finally:
with Finally(f"drop the table {name}"):
@@ -561,19 +570,27 @@ def create_partitioned_table_with_compact_and_wide_parts(
min_rows_for_wide_part=10,
min_bytes_for_wide_part=100,
engine="MergeTree",
+ columns=None,
+ partition_by="p",
+ cluster=None,
+ stop_merges=False,
):
"""Create a partitioned table that has specific settings in order
to get both wide and compact parts."""
+ if columns is None:
+ columns = [
+ Column(name="p", datatype=UInt8()),
+ Column(name="i", datatype=UInt64()),
+ ]
create_table(
name=table_name,
engine=engine,
- partition_by="p",
+ partition_by=partition_by,
order_by="tuple()",
- columns=[
- Column(name="p", datatype=UInt8()),
- Column(name="i", datatype=UInt64()),
- ],
+ columns=columns,
+ cluster=cluster,
query_settings=f"min_rows_for_wide_part={min_rows_for_wide_part}, min_bytes_for_wide_part={min_bytes_for_wide_part}",
+ stop_merges=stop_merges,
)
@@ -631,7 +648,9 @@ def attach_table(self, engine, columns, name=None, path=None, drop_sync=False):
"""
)
- yield Table(name, columns, engine)
+ yield Table(
+ name=name, columns=columns, partition_by=None, engine=engine
+ )
finally:
with Finally(f"drop the table {name}"):
diff --git a/s3/configs/clickhouse/config.d/storage.xml b/s3/configs/clickhouse/config.d/storage.xml
new file mode 100644
index 000000000..822eb90a7
--- /dev/null
+++ b/s3/configs/clickhouse/config.d/storage.xml
@@ -0,0 +1,82 @@
+
+
+
+
+ /jbod1/
+
+
+ /jbod2/
+
+
+ /jbod3/
+
+
+ /jbod4/
+
+
+ /external/
+
+
+ /external2/
+
+
+
+
+
+
+ jbod1
+
+
+
+
+
+
+ jbod2
+
+
+
+
+
+
+ jbod3
+
+
+
+
+
+
+ jbod4
+
+
+
+
+
+
+ external
+
+
+
+
+
+
+ external2
+
+
+
+
+
+
+ jbod1
+ jbod2
+ 2048
+
+
+ external
+ external2
+
+
+ 0.7
+
+
+
+
\ No newline at end of file
diff --git a/s3/configs/clickhouse/config.xml b/s3/configs/clickhouse/config.xml
index 0fee8bc4b..c40c589a9 100644
--- a/s3/configs/clickhouse/config.xml
+++ b/s3/configs/clickhouse/config.xml
@@ -416,7 +416,7 @@
86400
- 60
+ 7200
diff --git a/s3/regression.py b/s3/regression.py
index ac1500533..049f8cb7f 100755
--- a/s3/regression.py
+++ b/s3/regression.py
@@ -501,6 +501,16 @@
"doesn't work <22.8",
check_clickhouse_version("<22.8"),
),
+ "/:/:/part 3/export part/*": (
+ Skip,
+ "Export part introduced in Antalya build",
+ check_if_not_antalya_build,
+ ),
+ "/:/:/part 3/export partition/*": (
+ Skip,
+ "Export partition introduced in Antalya build",
+ check_if_not_antalya_build,
+ ),
}
@@ -539,6 +549,9 @@ def minio_regression(
self.context.cluster = cluster
self.context.node = cluster.node("clickhouse1")
+ self.context.node2 = cluster.node("clickhouse2")
+ self.context.node3 = cluster.node("clickhouse3")
+ self.context.nodes = [self.context.node, self.context.node2, self.context.node3]
with And("I have a minio client"):
start_minio(access_key=root_user, secret_key=root_password)
@@ -549,6 +562,10 @@ def minio_regression(
for node in nodes["clickhouse"]:
experimental_analyzer(node=cluster.node(node), with_analyzer=with_analyzer)
+ # with And("I install tc-netem on all clickhouse nodes"):
+ # for node in self.context.nodes:
+ # node.command("apt install --yes iproute2 procps")
+
with And("allow higher cpu_wait_ratio "):
if check_clickhouse_version(">=25.4")(self):
allow_higher_cpu_wait_ratio(
@@ -603,6 +620,12 @@ def minio_regression(
Feature(test=load("s3.tests.remote_s3_function", "minio"))(
uri=uri_bucket_file, bucket_prefix=bucket_prefix
)
+ Feature(test=load("s3.tests.export_part.feature", "minio"))(
+ uri=uri_bucket_file, bucket_prefix=bucket_prefix
+ )
+ Feature(test=load("s3.tests.export_partition.feature", "minio"))(
+ uri=uri_bucket_file, bucket_prefix=bucket_prefix
+ )
@TestFeature
diff --git a/s3/requirements/export_part.md b/s3/requirements/export_part.md
index 6e1bb8d67..c7675153a 100644
--- a/s3/requirements/export_part.md
+++ b/s3/requirements/export_part.md
@@ -6,52 +6,62 @@
* 1 [Introduction](#introduction)
* 2 [Exporting Parts to S3](#exporting-parts-to-s3)
* 2.1 [RQ.ClickHouse.ExportPart.S3](#rqclickhouseexportparts3)
+ * 2.2 [RQ.ClickHouse.ExportPart.EmptyTable](#rqclickhouseexportpartemptytable)
* 3 [SQL command support](#sql-command-support)
* 3.1 [RQ.ClickHouse.ExportPart.SQLCommand](#rqclickhouseexportpartsqlcommand)
* 4 [Supported source table engines](#supported-source-table-engines)
* 4.1 [RQ.ClickHouse.ExportPart.SourceEngines](#rqclickhouseexportpartsourceengines)
-* 5 [Supported source part storage types](#supported-source-part-storage-types)
- * 5.1 [RQ.ClickHouse.ExportPart.SourcePartStorage](#rqclickhouseexportpartSourcepartstorage)
-* 6 [Supported destination table engines](#supported-destination-table-engines)
- * 6.1 [RQ.ClickHouse.ExportPart.DestinationEngines](#rqclickhouseexportpartdestinationengines)
-* 7 [Destination setup and file management](#destination-setup-and-file-management)
- * 7.1 [RQ.ClickHouse.ExportPart.DestinationSetup](#rqclickhouseexportpartdestinationsetup)
-* 8 [Export data preparation](#export-data-preparation)
- * 8.1 [RQ.ClickHouse.ExportPart.DataPreparation](#rqclickhouseexportpartdatapreparation)
+* 5 [Cluster and node support](#cluster-and-node-support)
+ * 5.1 [RQ.ClickHouse.ExportPart.ClustersNodes](#rqclickhouseexportpartclustersnodes)
+* 6 [Supported source part storage types](#supported-source-part-storage-types)
+ * 6.1 [RQ.ClickHouse.ExportPart.SourcePartStorage](#rqclickhouseexportpartsourcepartstorage)
+* 7 [Storage policies and volumes](#storage-policies-and-volumes)
+ * 7.1 [RQ.ClickHouse.ExportPart.StoragePolicies](#rqclickhouseexportpartstoragepolicies)
+* 8 [Supported destination table engines](#supported-destination-table-engines)
+ * 8.1 [RQ.ClickHouse.ExportPart.DestinationEngines](#rqclickhouseexportpartdestinationengines)
* 9 [Schema compatibility](#schema-compatibility)
* 9.1 [RQ.ClickHouse.ExportPart.SchemaCompatibility](#rqclickhouseexportpartschemacompatibility)
* 10 [Partition key types support](#partition-key-types-support)
* 10.1 [RQ.ClickHouse.ExportPart.PartitionKeyTypes](#rqclickhouseexportpartpartitionkeytypes)
* 11 [Part types and content support](#part-types-and-content-support)
* 11.1 [RQ.ClickHouse.ExportPart.PartTypes](#rqclickhouseexportpartparttypes)
+ * 11.2 [RQ.ClickHouse.ExportPart.SchemaChangeIsolation](#rqclickhouseexportpartschemachangeisolation)
+ * 11.3 [RQ.ClickHouse.ExportPart.LargeParts](#rqclickhouseexportpartlargeparts)
* 12 [Export operation failure handling](#export-operation-failure-handling)
* 12.1 [RQ.ClickHouse.ExportPart.FailureHandling](#rqclickhouseexportpartfailurehandling)
-* 13 [Export operation restrictions](#export-operation-restrictions)
- * 13.1 [Preventing same table exports](#preventing-same-table-exports)
- * 13.1.1 [RQ.ClickHouse.ExportPart.Restrictions.SameTable](#rqclickhouseexportpartrestrictionssametable)
- * 13.2 [Destination table compatibility](#destination-table-compatibility)
- * 13.2.1 [RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport](#rqclickhouseexportpartrestrictionsdestinationsupport)
- * 13.3 [Source part availability](#source-part-availability)
- * 13.3.1 [RQ.ClickHouse.ExportPart.Restrictions.SourcePart](#rqclickhouseexportpartrestrictionssourcepart)
-* 14 [Export operation concurrency](#export-operation-concurrency)
- * 14.1 [RQ.ClickHouse.ExportPart.Concurrency](#rqclickhouseexportpartconcurrency)
-* 15 [Export operation idempotency](#export-operation-idempotency)
- * 15.1 [RQ.ClickHouse.ExportPart.Idempotency](#rqclickhouseexportpartidempotency)
-* 16 [Export operation logging](#export-operation-logging)
- * 16.1 [RQ.ClickHouse.ExportPart.Logging](#rqclickhouseexportpartlogging)
-* 17 [Monitoring export operations](#monitoring-export-operations)
- * 17.1 [RQ.ClickHouse.ExportPart.SystemTables.Exports](#rqclickhouseexportpartsystemtablesexports)
-* 18 [Enabling export functionality](#enabling-export-functionality)
- * 18.1 [RQ.ClickHouse.ExportPart.Settings.AllowExperimental](#rqclickhouseexportpartsettingsallowexperimental)
-* 19 [Handling file conflicts during export](#handling-file-conflicts-during-export)
- * 19.1 [RQ.ClickHouse.ExportPart.Settings.OverwriteFile](#rqclickhouseexportpartsettingsoverwritefile)
-* 20 [Export operation configuration](#export-operation-configuration)
- * 20.1 [RQ.ClickHouse.ExportPart.ParallelFormatting](#rqclickhouseexportpartparallelformatting)
-* 21 [Controlling export performance](#controlling-export-performance)
- * 21.1 [RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth](#rqclickhouseexportpartserversettingsmaxbandwidth)
-* 22 [Monitoring export performance metrics](#monitoring-export-performance-metrics)
- * 22.1 [RQ.ClickHouse.ExportPart.Events](#rqclickhouseexportpartevents)
- * 22.2 [RQ.ClickHouse.ExportPart.Metrics.Export](#rqclickhouseexportpartmetricsexport)
+* 13 [Network resilience](#network-resilience)
+ * 13.1 [RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues](#rqclickhouseexportpartnetworkresiliencepacketissues)
+ * 13.2 [RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption](#rqclickhouseexportpartnetworkresiliencedestinationinterruption)
+ * 13.3 [RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption](#rqclickhouseexportpartnetworkresiliencenodeinterruption)
+* 14 [Export operation restrictions](#export-operation-restrictions)
+ * 14.1 [Preventing same table exports](#preventing-same-table-exports)
+ * 14.1.1 [RQ.ClickHouse.ExportPart.Restrictions.SameTable](#rqclickhouseexportpartrestrictionssametable)
+ * 14.2 [Destination table compatibility](#destination-table-compatibility)
+ * 14.2.1 [RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport](#rqclickhouseexportpartrestrictionsdestinationsupport)
+ * 14.3 [Local table restriction](#local-table-restriction)
+ * 14.3.1 [RQ.ClickHouse.ExportPart.Restrictions.LocalTable](#rqclickhouseexportpartrestrictionslocaltable)
+ * 14.4 [Partition key compatibility](#partition-key-compatibility)
+ * 14.4.1 [RQ.ClickHouse.ExportPart.Restrictions.PartitionKey](#rqclickhouseexportpartrestrictionspartitionkey)
+ * 14.5 [Source part availability](#source-part-availability)
+ * 14.5.1 [RQ.ClickHouse.ExportPart.Restrictions.SourcePart](#rqclickhouseexportpartrestrictionssourcepart)
+* 15 [Export operation concurrency](#export-operation-concurrency)
+ * 15.1 [RQ.ClickHouse.ExportPart.Concurrency](#rqclickhouseexportpartconcurrency)
+* 16 [Export operation idempotency](#export-operation-idempotency)
+ * 16.1 [RQ.ClickHouse.ExportPart.Idempotency](#rqclickhouseexportpartidempotency)
+* 17 [Export operation logging](#export-operation-logging)
+ * 17.1 [RQ.ClickHouse.ExportPart.Logging](#rqclickhouseexportpartlogging)
+* 18 [Monitoring export operations](#monitoring-export-operations)
+ * 18.1 [RQ.ClickHouse.ExportPart.SystemTables.Exports](#rqclickhouseexportpartsystemtablesexports)
+* 19 [Enabling export functionality](#enabling-export-functionality)
+ * 19.1 [RQ.ClickHouse.ExportPart.Settings.AllowExperimental](#rqclickhouseexportpartsettingsallowexperimental)
+* 20 [Handling file conflicts during export](#handling-file-conflicts-during-export)
+ * 20.1 [RQ.ClickHouse.ExportPart.Settings.OverwriteFile](#rqclickhouseexportpartsettingsoverwritefile)
+* 21 [Export operation configuration](#export-operation-configuration)
+ * 21.1 [RQ.ClickHouse.ExportPart.ParallelFormatting](#rqclickhouseexportpartparallelformatting)
+* 22 [Controlling export performance](#controlling-export-performance)
+ * 22.1 [RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth](#rqclickhouseexportpartserversettingsmaxbandwidth)
+ * 22.2 [RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize](#rqclickhouseexportpartserversettingsbackgroundmovepoolsize)
+ * 22.3 [RQ.ClickHouse.ExportPart.Metrics.Export](#rqclickhouseexportpartmetricsexport)
* 23 [Export operation security](#export-operation-security)
* 23.1 [RQ.ClickHouse.ExportPart.Security](#rqclickhouseexportpartsecurity)
@@ -66,6 +76,15 @@ version: 1.0
[ClickHouse] SHALL support exporting data parts from MergeTree engine tables to S3 object storage.
+### RQ.ClickHouse.ExportPart.EmptyTable
+version: 1.0
+
+[ClickHouse] SHALL support exporting from empty tables by:
+* Completing export operations successfully when the source table contains no parts
+* Resulting in an empty destination table when exporting from an empty source table
+* Not creating any files in destination storage when there are no parts to export
+* Handling empty tables gracefully without errors
+
## SQL command support
### RQ.ClickHouse.ExportPart.SQLCommand
@@ -99,6 +118,16 @@ version: 1.0
* `GraphiteMergeTree` - MergeTree optimized for Graphite data
* All other MergeTree family engines that inherit from `MergeTreeData`
+## Cluster and node support
+
+### RQ.ClickHouse.ExportPart.ClustersNodes
+version: 1.0
+
+[ClickHouse] SHALL support exporting parts from multiple nodes in a cluster to the same destination storage, ensuring that:
+* Each node can independently export parts from its local storage to the shared destination
+* Exported data from different nodes is correctly aggregated in the destination
+* All nodes in the cluster can read the same exported data from the destination
+
## Supported source part storage types
### RQ.ClickHouse.ExportPart.SourcePartStorage
@@ -113,6 +142,19 @@ version: 1.0
* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)
* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled
+## Storage policies and volumes
+
+### RQ.ClickHouse.ExportPart.StoragePolicies
+version: 1.0
+
+[ClickHouse] SHALL support exporting parts from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:
+* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks
+* **External Volumes**: Volumes using external storage systems
+* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers
+* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks
+* Exporting parts regardless of which volume or disk within the storage policy contains the part
+* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy
+
## Supported destination table engines
### RQ.ClickHouse.ExportPart.DestinationEngines
@@ -125,28 +167,6 @@ version: 1.0
* `HDFS` - Hadoop Distributed File System (with Hive partitioning)
* `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)
* `GCS` - Google Cloud Storage (with Hive partitioning)
-* Implement the `supportsImport()` method and return `true`
-
-## Destination setup and file management
-
-### RQ.ClickHouse.ExportPart.DestinationSetup
-version: 1.0
-
-[ClickHouse] SHALL handle destination setup and file management by:
-* Creating appropriate import sinks for destination storage systems
-* Generating unique file names in the format `{part_name}_{checksum_hex}` to avoid conflicts
-* Allowing destination storage to determine the final file path based on Hive partitioning
-* Creating files in the destination storage that users can observe and access
-* Providing the final destination file path in the `system.exports` table for monitoring
-
-## Export data preparation
-
-### RQ.ClickHouse.ExportPart.DataPreparation
-version: 1.0
-
-[ClickHouse] SHALL prepare data for export by:
-* Automatically selecting all physical columns from source table metadata
-* Extracting partition key values for proper Hive partitioning in destination
## Schema compatibility
@@ -169,17 +189,9 @@ version: 1.0
| Partition Key Type | Supported | Examples | Notes |
|-------------------|------------|----------|-------|
| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |
-| **Date/DateTime Types** | ✅ Yes | `Date`, `DateTime`, `DateTime64` | All date/time types supported |
+| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |
| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |
-| **Date Functions** | ✅ Yes | `toYYYYMM(date_col)`, `toMonday(date_col)`, `toYear(date_col)` | Result in supported types |
-| **Mathematical Expressions** | ✅ Yes | `column1 + column2`, `column * 1000` | If result is supported type |
-| **String Functions** | ✅ Yes | `substring(column, 1, 4)` | Result in String type |
-| **Tuple Expressions** | ✅ Yes | `(toMonday(StartDate), EventType)` | If all elements are supported types |
| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |
-| **UUID Types** | ❌ No | `UUID` | Not supported by Hive partitioning |
-| **Enum Types** | ❌ No | `Enum8`, `Enum16` | Not supported by Hive partitioning |
-| **Floating-point Types** | ❌ No | `Float32`, `Float64` | Not supported by Hive partitioning |
-| **Hash Functions** | ❌ No | `intHash32(column)`, `cityHash64(column)` | Result in unsupported types |
[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.
@@ -196,38 +208,27 @@ version: 1.0
|-----------|------------|-------------|------------------|
| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |
| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |
-| **Regular Parts** | ✅ Yes | Standard data parts created by inserts, merges, mutations | Full data content |
-| **Patch Parts** | ✅ Yes | Lightweight update parts containing only changed columns | Applied during export |
-| **Active Parts** | ✅ Yes | Currently active data parts | Primary export target |
-| **Outdated Parts** | ✅ Yes | Parts that have been replaced by newer versions | Can be exported for backup |
-
-[ClickHouse] SHALL handle all special columns and metadata present in parts during export:
-
-| Column Type | Supported | Description | Export Behavior |
-|-------------|------------|-------------|-----------------|
-| **Physical Columns** | ✅ Yes | User-defined table columns | All physical columns exported |
-| **RowExistsColumn (_row_exists)** | ✅ Yes | Lightweight delete mask showing row existence | Exported to maintain delete state |
-| **BlockNumberColumn (_block_number)** | ✅ Yes | Original block number from insert | Exported for row identification |
-| **BlockOffsetColumn (_block_offset)** | ✅ Yes | Original row offset within block | Exported for row identification |
-| **PartDataVersionColumn (_part_data_version)** | ✅ Yes | Data version for mutations | Exported for version tracking |
-| **Virtual Columns** | ✅ Yes | Runtime columns like _part, _partition_id | Generated during export |
-| **System Metadata** | ✅ Yes | Checksums, compression info, serialization | Preserved in export |
-
-[ClickHouse] SHALL handle all mutation and schema change information present in parts:
-
-| Mutation/Schema Type | Supported | Description | Export Behavior |
-|---------------------|------------|-------------|-----------------|
-| **Mutation Commands** | ✅ Yes | DELETE, UPDATE, MATERIALIZE_INDEX, DROP_COLUMN, RENAME_COLUMN | Applied during export |
-| **Alter Conversions** | ✅ Yes | Column renames, type changes, schema modifications | Applied during export |
-| **Patch Parts** | ✅ Yes | Lightweight updates with only changed columns | Applied during export |
-| **Mutation Versions** | ✅ Yes | Version tracking for applied mutations | Preserved in export |
-| **Schema Changes** | ✅ Yes | ALTER MODIFY, ALTER DROP, ALTER RENAME | Applied during export |
-| **TTL Information** | ✅ Yes | Time-to-live settings and expiration data | Preserved in export |
-| **Index Information** | ✅ Yes | Primary key, secondary indices, projections | Preserved in export |
-| **Statistics** | ✅ Yes | Column statistics and sampling information | Preserved in export |
[ClickHouse] SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL handle all part metadata including checksums, compression information, serialization details, mutation history, schema changes, and structural modifications to maintain data integrity in the destination storage.
+### RQ.ClickHouse.ExportPart.SchemaChangeIsolation
+version: 1.0
+
+[ClickHouse] SHALL ensure exported data is isolated from subsequent schema changes by:
+* Preserving exported data exactly as it was at the time of export
+* Not being affected by schema changes (column drops, renames, type changes) that occur after export
+* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export
+* Ensuring exported data reflects the source table state at the time of export, not the current state
+
+### RQ.ClickHouse.ExportPart.LargeParts
+version: 1.0
+
+[ClickHouse] SHALL support exporting large parts by:
+* Handling parts with large numbers of rows (e.g., 100 million or more)
+* Processing large data volumes efficiently during export
+* Maintaining data integrity when exporting large parts
+* Completing export operations successfully regardless of part size
+
## Export operation failure handling
### RQ.ClickHouse.ExportPart.FailureHandling
@@ -240,6 +241,40 @@ version: 1.0
* **Simple Failure**: Export operations either succeed completely or fail with an error message
* **No Partial Exports**: Failed exports leave no partial or corrupted data in destination storage
+## Network resilience
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues
+version: 1.0
+
+[ClickHouse] SHALL handle network packet issues during export operations by:
+* Tolerating packet delay without data corruption or loss
+* Handling packet loss and retransmitting data as needed
+* Detecting and handling packet corruption to ensure data integrity
+* Managing packet duplication without data duplication in destination
+* Handling packet reordering to maintain correct data sequence
+* Operating correctly under packet rate limiting constraints
+* Completing exports successfully despite network impairments
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle destination storage interruptions during export operations by:
+* Detecting when destination storage becomes unavailable during export
+* Failing export operations gracefully when destination storage is unavailable
+* Logging failed exports in the `system.events` table with `PartsExportFailures` counter
+* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability
+* Allowing exports to complete successfully once destination storage becomes available again
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:
+* Handling node restarts gracefully during export operations
+* Not leaving partial or corrupted data in destination storage when node restarts occur
+* With safe shutdown, ensuring exports complete successfully before node shutdown
+* With unsafe shutdown, allowing partial exports to complete successfully after node restart
+* Maintaining data integrity in destination storage regardless of node interruption type
+
## Export operation restrictions
### Preventing same table exports
@@ -264,16 +299,35 @@ version: 1.0
* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met
* Performing this validation during the initial export setup phase
+### Local table restriction
+
+#### RQ.ClickHouse.ExportPart.Restrictions.LocalTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting parts to local MergeTree tables by:
+* Rejecting export operations where the destination table uses a MergeTree engine
+* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table
+* Performing this validation during the initial export setup phase
+
+### Partition key compatibility
+
+#### RQ.ClickHouse.ExportPart.Restrictions.PartitionKey
+version: 1.0
+
+[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:
+* Checking that the partition key expression matches between source and destination tables
+* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ
+* Performing this validation during the initial export setup phase
+
### Source part availability
#### RQ.ClickHouse.ExportPart.Restrictions.SourcePart
version: 1.0
[ClickHouse] SHALL validate source part availability by:
-
* Checking that the specified part exists in the source table
* Verifying the part is in an active state (not detached or missing)
-* Throwing a `NO_SUCH_DATA_PART` exception with message "No such data part '{}' to export in table '{}'" when the part is not found
+* Throwing an exception with message containing "Unexpected part name" when the part is not found
* Performing this validation before creating the export manifest
## Export operation concurrency
@@ -296,12 +350,11 @@ version: 1.0
### RQ.ClickHouse.ExportPart.Idempotency
version: 1.0
-[ClickHouse] SHALL ensure export operations are idempotent by:
-
-* Allowing the same part to be exported multiple times safely without data corruption
-* Supporting file overwrite control through the `export_merge_tree_part_overwrite_file_if_exists` setting
-* Generating unique file names using part name and checksum to avoid conflicts
-* Maintaining export state consistency across retries
+[ClickHouse] SHALL handle duplicate export operations by:
+* Preventing duplicate data from being exported when the same part is exported multiple times to the same destination
+* Detecting when an export operation attempts to export a part that already exists in the destination
+* Logging duplicate export attempts in the `system.events` table with the `PartsExportDuplicated` counter
+* Ensuring that destination data matches source data without duplication when the same part is exported multiple times
## Export operation logging
@@ -310,17 +363,12 @@ version: 1.0
[ClickHouse] SHALL provide detailed logging for export operations by:
* Logging all export operations (both successful and failed) with timestamps and details
-* Recording the specific part name and destination for all operations
-* Including execution time and progress information for all operations
-* Writing operation information to the `system.part_log` table with the following columns:
- * `event_type` - Set to `ExportPart` for export operations
- * `event_time` - Timestamp when the export operation occurred
- * `table` - Source table name
- * `part_name` - Name of the part being exported
- * `path_on_disk` - Path to the part in source storage
- * `duration_ms` - Execution time in milliseconds
- * `error` - Error message if the export failed (empty for successful exports)
- * `thread_id` - Thread ID performing the export
+* Recording the specific part name in the `system.part_log` table for all operations
+* Logging export events in the `system.events` table, including:
+ * `PartsExports` - Number of successful part exports
+ * `PartsExportFailures` - Number of failed part exports
+ * `PartsExportDuplicated` - Number of part exports that failed because target already exists
+* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PART`
* Providing sufficient detail for monitoring and troubleshooting export operations
## Monitoring export operations
@@ -328,18 +376,11 @@ version: 1.0
### RQ.ClickHouse.ExportPart.SystemTables.Exports
version: 1.0
-[ClickHouse] SHALL provide a `system.exports` table that allows users to monitor active and completed export operations, track progress metrics, performance statistics, and troubleshoot export issues with the following columns:
+[ClickHouse] SHALL provide a `system.exports` table that allows users to monitor active export operations with at least the following columns:
+* `source_table` - source table identifier
+* `destination_table` - destination table identifier
-* `source_database`, `source_table` - source table identifiers
-* `destination_database`, `destination_table` - destination table identifiers
-* `create_time` - when export was submitted
-* `part_name` - name of the exported part
-* `destination_file_path` - path in destination storage
-* `elapsed` - execution time in seconds
-* `rows_read`, `total_rows_to_read` - progress metrics
-* `total_size_bytes_compressed`, `total_size_bytes_uncompressed` - size metrics
-* `bytes_read_uncompressed` - bytes processed
-* `memory_usage`, `peak_memory_usage` - memory consumption
+The table SHALL track export operations before they complete and SHALL be empty after all exports complete.
## Enabling export functionality
@@ -373,18 +414,10 @@ version: 1.0
[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.
-## Monitoring export performance metrics
-
-### RQ.ClickHouse.ExportPart.Events
+### RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize
version: 1.0
-[ClickHouse] SHALL provide the following export-related events in the `system.events` table:
-* `PartsExports` - Number of successful part exports
-* `PartsExportFailures` - Number of failed part exports
-* `PartsExportDuplicated` - Number of part exports that failed because target already exists
-* `PartsExportTotalMilliseconds` - Total time spent on part export operations in milliseconds
-* `ExportsThrottlerBytes` - Bytes passed through the exports throttler
-* `ExportsThrottlerSleepMicroseconds` - Total time queries were sleeping to conform to export bandwidth throttling
+[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.
### RQ.ClickHouse.ExportPart.Metrics.Export
version: 1.0
@@ -407,4 +440,4 @@ version: 1.0
* **Credential Management**: Export operations must use secure credential storage and avoid exposing credentials in logs
-[ClickHouse]: https://clickhouse.com
\ No newline at end of file
+[ClickHouse]: https://clickhouse.com
diff --git a/s3/requirements/export_part.py b/s3/requirements/export_part.py
new file mode 100644
index 000000000..69f8e25a5
--- /dev/null
+++ b/s3/requirements/export_part.py
@@ -0,0 +1,1305 @@
+# These requirements were auto generated
+# from software requirements specification (SRS)
+# document by TestFlows v2.0.250110.1002922.
+# Do not edit by hand but re-generate instead
+# using 'tfs requirements generate' command.
+from testflows.core import Specification
+from testflows.core import Requirement
+
+Heading = Specification.Heading
+
+RQ_ClickHouse_ExportPart_S3 = Requirement(
+ name="RQ.ClickHouse.ExportPart.S3",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting data parts from MergeTree engine tables to S3 object storage.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="2.1",
+)
+
+RQ_ClickHouse_ExportPart_EmptyTable = Requirement(
+ name="RQ.ClickHouse.ExportPart.EmptyTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting from empty tables by:\n"
+ "* Completing export operations successfully when the source table contains no parts\n"
+ "* Resulting in an empty destination table when exporting from an empty source table\n"
+ "* Not creating any files in destination storage when there are no parts to export\n"
+ "* Handling empty tables gracefully without errors\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="2.2",
+)
+
+RQ_ClickHouse_ExportPart_SQLCommand = Requirement(
+ name="RQ.ClickHouse.ExportPart.SQLCommand",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the following SQL command syntax for exporting MergeTree data parts to object storage tables:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE [database.]source_table_name \n"
+ "EXPORT PART 'part_name' \n"
+ "TO TABLE [database.]destination_table_name\n"
+ "```\n"
+ "\n"
+ "**Parameters:**\n"
+ "- `source_table_name`: Name of the source MergeTree table\n"
+ "- `part_name`: Name of the specific part to export (string literal)\n"
+ "- `destination_table_name`: Name of the destination object storage table\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="3.1",
+)
+
+RQ_ClickHouse_ExportPart_SourceEngines = Requirement(
+ name="RQ.ClickHouse.ExportPart.SourceEngines",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting from the following source table engines:\n"
+ "* `MergeTree` - Base MergeTree engine\n"
+ "* `ReplicatedMergeTree` - Replicated MergeTree engine with ZooKeeper coordination\n"
+ "* `SummingMergeTree` - MergeTree with automatic summation of numeric columns\n"
+ "* `AggregatingMergeTree` - MergeTree with pre-aggregated data\n"
+ "* `CollapsingMergeTree` - MergeTree with row versioning for updates\n"
+ "* `VersionedCollapsingMergeTree` - CollapsingMergeTree with version tracking\n"
+ "* `GraphiteMergeTree` - MergeTree optimized for Graphite data\n"
+ "* All other MergeTree family engines that inherit from `MergeTreeData`\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="4.1",
+)
+
+RQ_ClickHouse_ExportPart_ClustersNodes = Requirement(
+ name="RQ.ClickHouse.ExportPart.ClustersNodes",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting parts from multiple nodes in a cluster to the same destination storage, ensuring that:\n"
+ "* Each node can independently export parts from its local storage to the shared destination\n"
+ "* Exported data from different nodes is correctly aggregated in the destination\n"
+ "* All nodes in the cluster can read the same exported data from the destination\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="5.1",
+)
+
+RQ_ClickHouse_ExportPart_SourcePartStorage = Requirement(
+ name="RQ.ClickHouse.ExportPart.SourcePartStorage",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting data parts regardless of the underlying storage type where the source parts are stored, including:\n"
+ "* **Local Disks**: Parts stored on local filesystem\n"
+ "* **S3/Object Storage**: Parts stored on S3 or S3-compatible object storage\n"
+ "* **Encrypted Disks**: Parts stored on encrypted disks (disk-level encryption)\n"
+ "* **Cached Disks**: Parts stored with filesystem cache enabled\n"
+ "* **Remote Disks**: Parts stored on HDFS, Azure Blob Storage, or Google Cloud Storage\n"
+ "* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)\n"
+ "* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="6.1",
+)
+
+RQ_ClickHouse_ExportPart_StoragePolicies = Requirement(
+ name="RQ.ClickHouse.ExportPart.StoragePolicies",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting parts from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:\n"
+ "* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks\n"
+ "* **External Volumes**: Volumes using external storage systems\n"
+ "* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers\n"
+ "* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks\n"
+ "* Exporting parts regardless of which volume or disk within the storage policy contains the part\n"
+ "* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="7.1",
+)
+
+RQ_ClickHouse_ExportPart_DestinationEngines = Requirement(
+ name="RQ.ClickHouse.ExportPart.DestinationEngines",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting to destination tables that:\n"
+ "* Support object storage engines including:\n"
+ " * `S3` - Amazon S3 and S3-compatible storage\n"
+ " * `StorageObjectStorage` - Generic object storage interface\n"
+ " * `HDFS` - Hadoop Distributed File System (with Hive partitioning)\n"
+ " * `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)\n"
+ " * `GCS` - Google Cloud Storage (with Hive partitioning)\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="8.1",
+)
+
+RQ_ClickHouse_ExportPart_SchemaCompatibility = Requirement(
+ name="RQ.ClickHouse.ExportPart.SchemaCompatibility",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL require source and destination tables to have compatible schemas for successful export operations:\n"
+ "* Identical physical column schemas between source and destination\n"
+ "* The same partition key expression in both tables\n"
+ "* Compatible data types for all columns\n"
+ "* Matching column order and names\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="9.1",
+)
+
+RQ_ClickHouse_ExportPart_PartitionKeyTypes = Requirement(
+ name="RQ.ClickHouse.ExportPart.PartitionKeyTypes",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support export operations for tables with partition key types that are compatible with Hive partitioning, as shown in the following table:\n"
+ "\n"
+ "| Partition Key Type | Supported | Examples | Notes |\n"
+ "|-------------------|------------|----------|-------|\n"
+ "| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |\n"
+ "| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |\n"
+ "| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |\n"
+ "| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |\n"
+ "\n"
+ "[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.\n"
+ "\n"
+ "[ClickHouse] SHALL require destination tables to support Hive partitioning, which limits the supported partition key types to Integer, Date/DateTime, and String types. Complex expressions that result in unsupported types are not supported for export operations.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="10.1",
+)
+
+RQ_ClickHouse_ExportPart_PartTypes = Requirement(
+ name="RQ.ClickHouse.ExportPart.PartTypes",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support export operations for all valid MergeTree part types and their contents, including:\n"
+ "\n"
+ "| Part Type | Supported | Description | Special Features |\n"
+ "|-----------|------------|-------------|------------------|\n"
+ "| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |\n"
+ "| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |\n"
+ "\n"
+ "[ClickHouse] SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL handle all part metadata including checksums, compression information, serialization details, mutation history, schema changes, and structural modifications to maintain data integrity in the destination storage.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="11.1",
+)
+
+RQ_ClickHouse_ExportPart_SchemaChangeIsolation = Requirement(
+ name="RQ.ClickHouse.ExportPart.SchemaChangeIsolation",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL ensure exported data is isolated from subsequent schema changes by:\n"
+ "* Preserving exported data exactly as it was at the time of export\n"
+ "* Not being affected by schema changes (column drops, renames, type changes) that occur after export\n"
+ "* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export\n"
+ "* Ensuring exported data reflects the source table state at the time of export, not the current state\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="11.2",
+)
+
+RQ_ClickHouse_ExportPart_LargeParts = Requirement(
+ name="RQ.ClickHouse.ExportPart.LargeParts",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting large parts by:\n"
+ "* Handling parts with large numbers of rows (e.g., 100 million or more)\n"
+ "* Processing large data volumes efficiently during export\n"
+ "* Maintaining data integrity when exporting large parts\n"
+ "* Completing export operations successfully regardless of part size\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="11.3",
+)
+
+RQ_ClickHouse_ExportPart_FailureHandling = Requirement(
+ name="RQ.ClickHouse.ExportPart.FailureHandling",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle export operation failures in the following ways:\n"
+ "* **Stateless Operation**: Export operations are stateless and ephemeral\n"
+ "* **No Recovery**: If an export fails, it fails completely with no retry mechanism\n"
+ "* **No State Persistence**: No export manifests or state are preserved across server restarts\n"
+ "* **Simple Failure**: Export operations either succeed completely or fail with an error message\n"
+ "* **No Partial Exports**: Failed exports leave no partial or corrupted data in destination storage\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="12.1",
+)
+
+RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues = Requirement(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle network packet issues during export operations by:\n"
+ "* Tolerating packet delay without data corruption or loss\n"
+ "* Handling packet loss and retransmitting data as needed\n"
+ "* Detecting and handling packet corruption to ensure data integrity\n"
+ "* Managing packet duplication without data duplication in destination\n"
+ "* Handling packet reordering to maintain correct data sequence\n"
+ "* Operating correctly under packet rate limiting constraints\n"
+ "* Completing exports successfully despite network impairments\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.1",
+)
+
+RQ_ClickHouse_ExportPart_NetworkResilience_DestinationInterruption = Requirement(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle destination storage interruptions during export operations by:\n"
+ "* Detecting when destination storage becomes unavailable during export\n"
+ "* Failing export operations gracefully when destination storage is unavailable\n"
+ "* Logging failed exports in the `system.events` table with `PartsExportFailures` counter\n"
+ "* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability\n"
+ "* Allowing exports to complete successfully once destination storage becomes available again\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.2",
+)
+
+RQ_ClickHouse_ExportPart_NetworkResilience_NodeInterruption = Requirement(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:\n"
+ "* Handling node restarts gracefully during export operations\n"
+ "* Not leaving partial or corrupted data in destination storage when node restarts occur\n"
+ "* With safe shutdown, ensuring exports complete successfully before node shutdown\n"
+ "* With unsafe shutdown, allowing partial exports to complete successfully after node restart\n"
+ "* Maintaining data integrity in destination storage regardless of node interruption type\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.3",
+)
+
+RQ_ClickHouse_ExportPart_Restrictions_SameTable = Requirement(
+ name="RQ.ClickHouse.ExportPart.Restrictions.SameTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL prevent exporting parts to the same table as the source by:\n"
+ "* Validating that source and destination table identifiers are different\n"
+ '* Throwing a `BAD_ARGUMENTS` exception with message "Exporting to the same table is not allowed" when source and destination are identical\n'
+ "* Performing this validation before any export processing begins\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="14.1.1",
+)
+
+RQ_ClickHouse_ExportPart_Restrictions_DestinationSupport = Requirement(
+ name="RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate destination table compatibility by:\n"
+ "\n"
+ "* Checking that the destination storage supports importing MergeTree parts\n"
+ "* Verifying that the destination uses Hive partitioning strategy (`partition_strategy = 'hive'`)\n"
+ '* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="14.2.1",
+)
+
+RQ_ClickHouse_ExportPart_Restrictions_LocalTable = Requirement(
+ name="RQ.ClickHouse.ExportPart.Restrictions.LocalTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL prevent exporting parts to local MergeTree tables by:\n"
+ "* Rejecting export operations where the destination table uses a MergeTree engine\n"
+ '* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="14.3.1",
+)
+
+RQ_ClickHouse_ExportPart_Restrictions_PartitionKey = Requirement(
+ name="RQ.ClickHouse.ExportPart.Restrictions.PartitionKey",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:\n"
+ "* Checking that the partition key expression matches between source and destination tables\n"
+ '* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="14.4.1",
+)
+
+RQ_ClickHouse_ExportPart_Restrictions_SourcePart = Requirement(
+ name="RQ.ClickHouse.ExportPart.Restrictions.SourcePart",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate source part availability by:\n"
+ "* Checking that the specified part exists in the source table\n"
+ "* Verifying the part is in an active state (not detached or missing)\n"
+ '* Throwing an exception with message containing "Unexpected part name" when the part is not found\n'
+ "* Performing this validation before creating the export manifest\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="14.5.1",
+)
+
+RQ_ClickHouse_ExportPart_Concurrency = Requirement(
+ name="RQ.ClickHouse.ExportPart.Concurrency",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support concurrent export operations by:\n"
+ "\n"
+ "* Allowing multiple exports to run simultaneously without interference\n"
+ "* Processing export operations asynchronously in the background\n"
+ "* Preventing race conditions and data corruption during concurrent operations\n"
+ "* Supporting concurrent exports of different parts to different destinations\n"
+ "* Preventing concurrent exports of the same part to the same destination\n"
+ "* Maintaining separate progress tracking and state for each concurrent operation\n"
+ "* Ensuring thread safety across all concurrent export operations\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="15.1",
+)
+
+RQ_ClickHouse_ExportPart_Idempotency = Requirement(
+ name="RQ.ClickHouse.ExportPart.Idempotency",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle duplicate export operations by:\n"
+ "* Preventing duplicate data from being exported when the same part is exported multiple times to the same destination\n"
+ "* Detecting when an export operation attempts to export a part that already exists in the destination\n"
+ "* Logging duplicate export attempts in the `system.events` table with the `PartsExportDuplicated` counter\n"
+ "* Ensuring that destination data matches source data without duplication when the same part is exported multiple times\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="16.1",
+)
+
+RQ_ClickHouse_ExportPart_Logging = Requirement(
+ name="RQ.ClickHouse.ExportPart.Logging",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide detailed logging for export operations by:\n"
+ "* Logging all export operations (both successful and failed) with timestamps and details\n"
+ "* Recording the specific part name in the `system.part_log` table for all operations\n"
+ "* Logging export events in the `system.events` table, including:\n"
+ " * `PartsExports` - Number of successful part exports\n"
+ " * `PartsExportFailures` - Number of failed part exports\n"
+ " * `PartsExportDuplicated` - Number of part exports that failed because target already exists\n"
+ "* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PART`\n"
+ "* Providing sufficient detail for monitoring and troubleshooting export operations\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="17.1",
+)
+
+RQ_ClickHouse_ExportPart_SystemTables_Exports = Requirement(
+ name="RQ.ClickHouse.ExportPart.SystemTables.Exports",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide a `system.exports` table that allows users to monitor active export operations with at least the following columns:\n"
+ "* `source_table` - source table identifier\n"
+ "* `destination_table` - destination table identifier\n"
+ "\n"
+ "The table SHALL track export operations before they complete and SHALL be empty after all exports complete.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="18.1",
+)
+
+RQ_ClickHouse_ExportPart_Settings_AllowExperimental = Requirement(
+ name="RQ.ClickHouse.ExportPart.Settings.AllowExperimental",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `allow_experimental_export_merge_tree_part` setting that SHALL gate the experimental export part functionality, which SHALL be set to `1` to enable `ALTER TABLE ... EXPORT PART ...` commands. The default value SHALL be `0` (turned off).\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="19.1",
+)
+
+RQ_ClickHouse_ExportPart_Settings_OverwriteFile = Requirement(
+ name="RQ.ClickHouse.ExportPart.Settings.OverwriteFile",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `export_merge_tree_part_overwrite_file_if_exists` setting that controls whether to overwrite files if they already exist when exporting a merge tree part. The default value SHALL be `0` (turned off).\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="20.1",
+)
+
+RQ_ClickHouse_ExportPart_ParallelFormatting = Requirement(
+ name="RQ.ClickHouse.ExportPart.ParallelFormatting",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support parallel formatting for export operations by:\n"
+ "* Automatically enabling parallel formatting for large export operations to improve performance\n"
+ "* Using the `output_format_parallel_formatting` setting to control parallel formatting behavior\n"
+ "* Optimizing data processing based on export size and system resources\n"
+ "* Providing consistent formatting performance across different export scenarios\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="21.1",
+)
+
+RQ_ClickHouse_ExportPart_ServerSettings_MaxBandwidth = Requirement(
+ name="RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="22.1",
+)
+
+RQ_ClickHouse_ExportPart_ServerSettings_BackgroundMovePoolSize = Requirement(
+ name="RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="22.2",
+)
+
+RQ_ClickHouse_ExportPart_Metrics_Export = Requirement(
+ name="RQ.ClickHouse.ExportPart.Metrics.Export",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide the `Export` current metric in `system.metrics` table that tracks the number of currently executing exports.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="22.3",
+)
+
+RQ_ClickHouse_ExportPart_Security = Requirement(
+ name="RQ.ClickHouse.ExportPart.Security",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL enforce security requirements for export operations:\n"
+ "* **RBAC**: Users must have the following privileges:\n"
+ " * **Source Table**: `SELECT` privilege on the source table to read data parts\n"
+ " * **Destination Table**: `INSERT` privilege on the destination table to write exported data\n"
+ " * **Database Access**: `SHOW` privilege on both source and destination databases\n"
+ " * **System Tables**: `SELECT` privilege on `system.tables` to validate table existence\n"
+ "* **Data Encryption**: All data in transit to destination storage must be encrypted using TLS/SSL\n"
+ "* **Network Security**: Export operations must use secure connections to destination storage (HTTPS for S3, secure protocols for other storage)\n"
+ "* **Credential Management**: Export operations must use secure credential storage and avoid exposing credentials in logs\n"
+ "\n"
+ "\n"
+ "[ClickHouse]: https://clickhouse.com\n"
+ ),
+ link=None,
+ level=2,
+ num="23.1",
+)
+
+SRS_015_ClickHouse_Export_Part_to_S3 = Specification(
+ name="SRS-015 ClickHouse Export Part to S3",
+ description=None,
+ author=None,
+ date=None,
+ status=None,
+ approved_by=None,
+ approved_date=None,
+ approved_version=None,
+ version=None,
+ group=None,
+ type=None,
+ link=None,
+ uid=None,
+ parent=None,
+ children=None,
+ headings=(
+ Heading(name="Introduction", level=1, num="1"),
+ Heading(name="Exporting Parts to S3", level=1, num="2"),
+ Heading(name="RQ.ClickHouse.ExportPart.S3", level=2, num="2.1"),
+ Heading(name="RQ.ClickHouse.ExportPart.EmptyTable", level=2, num="2.2"),
+ Heading(name="SQL command support", level=1, num="3"),
+ Heading(name="RQ.ClickHouse.ExportPart.SQLCommand", level=2, num="3.1"),
+ Heading(name="Supported source table engines", level=1, num="4"),
+ Heading(name="RQ.ClickHouse.ExportPart.SourceEngines", level=2, num="4.1"),
+ Heading(name="Cluster and node support", level=1, num="5"),
+ Heading(name="RQ.ClickHouse.ExportPart.ClustersNodes", level=2, num="5.1"),
+ Heading(name="Supported source part storage types", level=1, num="6"),
+ Heading(name="RQ.ClickHouse.ExportPart.SourcePartStorage", level=2, num="6.1"),
+ Heading(name="Storage policies and volumes", level=1, num="7"),
+ Heading(name="RQ.ClickHouse.ExportPart.StoragePolicies", level=2, num="7.1"),
+ Heading(name="Supported destination table engines", level=1, num="8"),
+ Heading(name="RQ.ClickHouse.ExportPart.DestinationEngines", level=2, num="8.1"),
+ Heading(name="Schema compatibility", level=1, num="9"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.SchemaCompatibility", level=2, num="9.1"
+ ),
+ Heading(name="Partition key types support", level=1, num="10"),
+ Heading(name="RQ.ClickHouse.ExportPart.PartitionKeyTypes", level=2, num="10.1"),
+ Heading(name="Part types and content support", level=1, num="11"),
+ Heading(name="RQ.ClickHouse.ExportPart.PartTypes", level=2, num="11.1"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.SchemaChangeIsolation", level=2, num="11.2"
+ ),
+ Heading(name="RQ.ClickHouse.ExportPart.LargeParts", level=2, num="11.3"),
+ Heading(name="Export operation failure handling", level=1, num="12"),
+ Heading(name="RQ.ClickHouse.ExportPart.FailureHandling", level=2, num="12.1"),
+ Heading(name="Network resilience", level=1, num="13"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues",
+ level=2,
+ num="13.1",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption",
+ level=2,
+ num="13.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption",
+ level=2,
+ num="13.3",
+ ),
+ Heading(name="Export operation restrictions", level=1, num="14"),
+ Heading(name="Preventing same table exports", level=2, num="14.1"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Restrictions.SameTable",
+ level=3,
+ num="14.1.1",
+ ),
+ Heading(name="Destination table compatibility", level=2, num="14.2"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport",
+ level=3,
+ num="14.2.1",
+ ),
+ Heading(name="Local table restriction", level=2, num="14.3"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Restrictions.LocalTable",
+ level=3,
+ num="14.3.1",
+ ),
+ Heading(name="Partition key compatibility", level=2, num="14.4"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Restrictions.PartitionKey",
+ level=3,
+ num="14.4.1",
+ ),
+ Heading(name="Source part availability", level=2, num="14.5"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Restrictions.SourcePart",
+ level=3,
+ num="14.5.1",
+ ),
+ Heading(name="Export operation concurrency", level=1, num="15"),
+ Heading(name="RQ.ClickHouse.ExportPart.Concurrency", level=2, num="15.1"),
+ Heading(name="Export operation idempotency", level=1, num="16"),
+ Heading(name="RQ.ClickHouse.ExportPart.Idempotency", level=2, num="16.1"),
+ Heading(name="Export operation logging", level=1, num="17"),
+ Heading(name="RQ.ClickHouse.ExportPart.Logging", level=2, num="17.1"),
+ Heading(name="Monitoring export operations", level=1, num="18"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.SystemTables.Exports", level=2, num="18.1"
+ ),
+ Heading(name="Enabling export functionality", level=1, num="19"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Settings.AllowExperimental",
+ level=2,
+ num="19.1",
+ ),
+ Heading(name="Handling file conflicts during export", level=1, num="20"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.Settings.OverwriteFile", level=2, num="20.1"
+ ),
+ Heading(name="Export operation configuration", level=1, num="21"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.ParallelFormatting", level=2, num="21.1"
+ ),
+ Heading(name="Controlling export performance", level=1, num="22"),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth",
+ level=2,
+ num="22.1",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize",
+ level=2,
+ num="22.2",
+ ),
+ Heading(name="RQ.ClickHouse.ExportPart.Metrics.Export", level=2, num="22.3"),
+ Heading(name="Export operation security", level=1, num="23"),
+ Heading(name="RQ.ClickHouse.ExportPart.Security", level=2, num="23.1"),
+ ),
+ requirements=(
+ RQ_ClickHouse_ExportPart_S3,
+ RQ_ClickHouse_ExportPart_EmptyTable,
+ RQ_ClickHouse_ExportPart_SQLCommand,
+ RQ_ClickHouse_ExportPart_SourceEngines,
+ RQ_ClickHouse_ExportPart_ClustersNodes,
+ RQ_ClickHouse_ExportPart_SourcePartStorage,
+ RQ_ClickHouse_ExportPart_StoragePolicies,
+ RQ_ClickHouse_ExportPart_DestinationEngines,
+ RQ_ClickHouse_ExportPart_SchemaCompatibility,
+ RQ_ClickHouse_ExportPart_PartitionKeyTypes,
+ RQ_ClickHouse_ExportPart_PartTypes,
+ RQ_ClickHouse_ExportPart_SchemaChangeIsolation,
+ RQ_ClickHouse_ExportPart_LargeParts,
+ RQ_ClickHouse_ExportPart_FailureHandling,
+ RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues,
+ RQ_ClickHouse_ExportPart_NetworkResilience_DestinationInterruption,
+ RQ_ClickHouse_ExportPart_NetworkResilience_NodeInterruption,
+ RQ_ClickHouse_ExportPart_Restrictions_SameTable,
+ RQ_ClickHouse_ExportPart_Restrictions_DestinationSupport,
+ RQ_ClickHouse_ExportPart_Restrictions_LocalTable,
+ RQ_ClickHouse_ExportPart_Restrictions_PartitionKey,
+ RQ_ClickHouse_ExportPart_Restrictions_SourcePart,
+ RQ_ClickHouse_ExportPart_Concurrency,
+ RQ_ClickHouse_ExportPart_Idempotency,
+ RQ_ClickHouse_ExportPart_Logging,
+ RQ_ClickHouse_ExportPart_SystemTables_Exports,
+ RQ_ClickHouse_ExportPart_Settings_AllowExperimental,
+ RQ_ClickHouse_ExportPart_Settings_OverwriteFile,
+ RQ_ClickHouse_ExportPart_ParallelFormatting,
+ RQ_ClickHouse_ExportPart_ServerSettings_MaxBandwidth,
+ RQ_ClickHouse_ExportPart_ServerSettings_BackgroundMovePoolSize,
+ RQ_ClickHouse_ExportPart_Metrics_Export,
+ RQ_ClickHouse_ExportPart_Security,
+ ),
+ content=r"""
+# SRS-015 ClickHouse Export Part to S3
+# Software Requirements Specification
+
+## Table of Contents
+
+* 1 [Introduction](#introduction)
+* 2 [Exporting Parts to S3](#exporting-parts-to-s3)
+ * 2.1 [RQ.ClickHouse.ExportPart.S3](#rqclickhouseexportparts3)
+ * 2.2 [RQ.ClickHouse.ExportPart.EmptyTable](#rqclickhouseexportpartemptytable)
+* 3 [SQL command support](#sql-command-support)
+ * 3.1 [RQ.ClickHouse.ExportPart.SQLCommand](#rqclickhouseexportpartsqlcommand)
+* 4 [Supported source table engines](#supported-source-table-engines)
+ * 4.1 [RQ.ClickHouse.ExportPart.SourceEngines](#rqclickhouseexportpartsourceengines)
+* 5 [Cluster and node support](#cluster-and-node-support)
+ * 5.1 [RQ.ClickHouse.ExportPart.ClustersNodes](#rqclickhouseexportpartclustersnodes)
+* 6 [Supported source part storage types](#supported-source-part-storage-types)
+ * 6.1 [RQ.ClickHouse.ExportPart.SourcePartStorage](#rqclickhouseexportpartsourcepartstorage)
+* 7 [Storage policies and volumes](#storage-policies-and-volumes)
+ * 7.1 [RQ.ClickHouse.ExportPart.StoragePolicies](#rqclickhouseexportpartstoragepolicies)
+* 8 [Supported destination table engines](#supported-destination-table-engines)
+ * 8.1 [RQ.ClickHouse.ExportPart.DestinationEngines](#rqclickhouseexportpartdestinationengines)
+* 9 [Schema compatibility](#schema-compatibility)
+ * 9.1 [RQ.ClickHouse.ExportPart.SchemaCompatibility](#rqclickhouseexportpartschemacompatibility)
+* 10 [Partition key types support](#partition-key-types-support)
+ * 10.1 [RQ.ClickHouse.ExportPart.PartitionKeyTypes](#rqclickhouseexportpartpartitionkeytypes)
+* 11 [Part types and content support](#part-types-and-content-support)
+ * 11.1 [RQ.ClickHouse.ExportPart.PartTypes](#rqclickhouseexportpartparttypes)
+ * 11.2 [RQ.ClickHouse.ExportPart.SchemaChangeIsolation](#rqclickhouseexportpartschemachangeisolation)
+ * 11.3 [RQ.ClickHouse.ExportPart.LargeParts](#rqclickhouseexportpartlargeparts)
+* 12 [Export operation failure handling](#export-operation-failure-handling)
+ * 12.1 [RQ.ClickHouse.ExportPart.FailureHandling](#rqclickhouseexportpartfailurehandling)
+* 13 [Network resilience](#network-resilience)
+ * 13.1 [RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues](#rqclickhouseexportpartnetworkresiliencepacketissues)
+ * 13.2 [RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption](#rqclickhouseexportpartnetworkresiliencedestinationinterruption)
+ * 13.3 [RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption](#rqclickhouseexportpartnetworkresiliencenodeinterruption)
+* 14 [Export operation restrictions](#export-operation-restrictions)
+ * 14.1 [Preventing same table exports](#preventing-same-table-exports)
+ * 14.1.1 [RQ.ClickHouse.ExportPart.Restrictions.SameTable](#rqclickhouseexportpartrestrictionssametable)
+ * 14.2 [Destination table compatibility](#destination-table-compatibility)
+ * 14.2.1 [RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport](#rqclickhouseexportpartrestrictionsdestinationsupport)
+ * 14.3 [Local table restriction](#local-table-restriction)
+ * 14.3.1 [RQ.ClickHouse.ExportPart.Restrictions.LocalTable](#rqclickhouseexportpartrestrictionslocaltable)
+ * 14.4 [Partition key compatibility](#partition-key-compatibility)
+ * 14.4.1 [RQ.ClickHouse.ExportPart.Restrictions.PartitionKey](#rqclickhouseexportpartrestrictionspartitionkey)
+ * 14.5 [Source part availability](#source-part-availability)
+ * 14.5.1 [RQ.ClickHouse.ExportPart.Restrictions.SourcePart](#rqclickhouseexportpartrestrictionssourcepart)
+* 15 [Export operation concurrency](#export-operation-concurrency)
+ * 15.1 [RQ.ClickHouse.ExportPart.Concurrency](#rqclickhouseexportpartconcurrency)
+* 16 [Export operation idempotency](#export-operation-idempotency)
+ * 16.1 [RQ.ClickHouse.ExportPart.Idempotency](#rqclickhouseexportpartidempotency)
+* 17 [Export operation logging](#export-operation-logging)
+ * 17.1 [RQ.ClickHouse.ExportPart.Logging](#rqclickhouseexportpartlogging)
+* 18 [Monitoring export operations](#monitoring-export-operations)
+ * 18.1 [RQ.ClickHouse.ExportPart.SystemTables.Exports](#rqclickhouseexportpartsystemtablesexports)
+* 19 [Enabling export functionality](#enabling-export-functionality)
+ * 19.1 [RQ.ClickHouse.ExportPart.Settings.AllowExperimental](#rqclickhouseexportpartsettingsallowexperimental)
+* 20 [Handling file conflicts during export](#handling-file-conflicts-during-export)
+ * 20.1 [RQ.ClickHouse.ExportPart.Settings.OverwriteFile](#rqclickhouseexportpartsettingsoverwritefile)
+* 21 [Export operation configuration](#export-operation-configuration)
+ * 21.1 [RQ.ClickHouse.ExportPart.ParallelFormatting](#rqclickhouseexportpartparallelformatting)
+* 22 [Controlling export performance](#controlling-export-performance)
+ * 22.1 [RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth](#rqclickhouseexportpartserversettingsmaxbandwidth)
+ * 22.2 [RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize](#rqclickhouseexportpartserversettingsbackgroundmovepoolsize)
+ * 22.3 [RQ.ClickHouse.ExportPart.Metrics.Export](#rqclickhouseexportpartmetricsexport)
+* 23 [Export operation security](#export-operation-security)
+ * 23.1 [RQ.ClickHouse.ExportPart.Security](#rqclickhouseexportpartsecurity)
+
+## Introduction
+
+This specification defines requirements for exporting individual MergeTree data parts to S3-compatible object storage.
+
+## Exporting Parts to S3
+
+### RQ.ClickHouse.ExportPart.S3
+version: 1.0
+
+[ClickHouse] SHALL support exporting data parts from MergeTree engine tables to S3 object storage.
+
+### RQ.ClickHouse.ExportPart.EmptyTable
+version: 1.0
+
+[ClickHouse] SHALL support exporting from empty tables by:
+* Completing export operations successfully when the source table contains no parts
+* Resulting in an empty destination table when exporting from an empty source table
+* Not creating any files in destination storage when there are no parts to export
+* Handling empty tables gracefully without errors
+
+## SQL command support
+
+### RQ.ClickHouse.ExportPart.SQLCommand
+version: 1.0
+
+[ClickHouse] SHALL support the following SQL command syntax for exporting MergeTree data parts to object storage tables:
+
+```sql
+ALTER TABLE [database.]source_table_name
+EXPORT PART 'part_name'
+TO TABLE [database.]destination_table_name
+```
+
+**Parameters:**
+- `source_table_name`: Name of the source MergeTree table
+- `part_name`: Name of the specific part to export (string literal)
+- `destination_table_name`: Name of the destination object storage table
+
+## Supported source table engines
+
+### RQ.ClickHouse.ExportPart.SourceEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting from the following source table engines:
+* `MergeTree` - Base MergeTree engine
+* `ReplicatedMergeTree` - Replicated MergeTree engine with ZooKeeper coordination
+* `SummingMergeTree` - MergeTree with automatic summation of numeric columns
+* `AggregatingMergeTree` - MergeTree with pre-aggregated data
+* `CollapsingMergeTree` - MergeTree with row versioning for updates
+* `VersionedCollapsingMergeTree` - CollapsingMergeTree with version tracking
+* `GraphiteMergeTree` - MergeTree optimized for Graphite data
+* All other MergeTree family engines that inherit from `MergeTreeData`
+
+## Cluster and node support
+
+### RQ.ClickHouse.ExportPart.ClustersNodes
+version: 1.0
+
+[ClickHouse] SHALL support exporting parts from multiple nodes in a cluster to the same destination storage, ensuring that:
+* Each node can independently export parts from its local storage to the shared destination
+* Exported data from different nodes is correctly aggregated in the destination
+* All nodes in the cluster can read the same exported data from the destination
+
+## Supported source part storage types
+
+### RQ.ClickHouse.ExportPart.SourcePartStorage
+version: 1.0
+
+[ClickHouse] SHALL support exporting data parts regardless of the underlying storage type where the source parts are stored, including:
+* **Local Disks**: Parts stored on local filesystem
+* **S3/Object Storage**: Parts stored on S3 or S3-compatible object storage
+* **Encrypted Disks**: Parts stored on encrypted disks (disk-level encryption)
+* **Cached Disks**: Parts stored with filesystem cache enabled
+* **Remote Disks**: Parts stored on HDFS, Azure Blob Storage, or Google Cloud Storage
+* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)
+* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled
+
+## Storage policies and volumes
+
+### RQ.ClickHouse.ExportPart.StoragePolicies
+version: 1.0
+
+[ClickHouse] SHALL support exporting parts from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:
+* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks
+* **External Volumes**: Volumes using external storage systems
+* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers
+* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks
+* Exporting parts regardless of which volume or disk within the storage policy contains the part
+* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy
+
+## Supported destination table engines
+
+### RQ.ClickHouse.ExportPart.DestinationEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting to destination tables that:
+* Support object storage engines including:
+ * `S3` - Amazon S3 and S3-compatible storage
+ * `StorageObjectStorage` - Generic object storage interface
+ * `HDFS` - Hadoop Distributed File System (with Hive partitioning)
+ * `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)
+ * `GCS` - Google Cloud Storage (with Hive partitioning)
+
+## Schema compatibility
+
+### RQ.ClickHouse.ExportPart.SchemaCompatibility
+version: 1.0
+
+[ClickHouse] SHALL require source and destination tables to have compatible schemas for successful export operations:
+* Identical physical column schemas between source and destination
+* The same partition key expression in both tables
+* Compatible data types for all columns
+* Matching column order and names
+
+## Partition key types support
+
+### RQ.ClickHouse.ExportPart.PartitionKeyTypes
+version: 1.0
+
+[ClickHouse] SHALL support export operations for tables with partition key types that are compatible with Hive partitioning, as shown in the following table:
+
+| Partition Key Type | Supported | Examples | Notes |
+|-------------------|------------|----------|-------|
+| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |
+| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |
+| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |
+| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |
+
+[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.
+
+[ClickHouse] SHALL require destination tables to support Hive partitioning, which limits the supported partition key types to Integer, Date/DateTime, and String types. Complex expressions that result in unsupported types are not supported for export operations.
+
+## Part types and content support
+
+### RQ.ClickHouse.ExportPart.PartTypes
+version: 1.0
+
+[ClickHouse] SHALL support export operations for all valid MergeTree part types and their contents, including:
+
+| Part Type | Supported | Description | Special Features |
+|-----------|------------|-------------|------------------|
+| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |
+| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |
+
+[ClickHouse] SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL handle all part metadata including checksums, compression information, serialization details, mutation history, schema changes, and structural modifications to maintain data integrity in the destination storage.
+
+### RQ.ClickHouse.ExportPart.SchemaChangeIsolation
+version: 1.0
+
+[ClickHouse] SHALL ensure exported data is isolated from subsequent schema changes by:
+* Preserving exported data exactly as it was at the time of export
+* Not being affected by schema changes (column drops, renames, type changes) that occur after export
+* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export
+* Ensuring exported data reflects the source table state at the time of export, not the current state
+
+### RQ.ClickHouse.ExportPart.LargeParts
+version: 1.0
+
+[ClickHouse] SHALL support exporting large parts by:
+* Handling parts with large numbers of rows (e.g., 100 million or more)
+* Processing large data volumes efficiently during export
+* Maintaining data integrity when exporting large parts
+* Completing export operations successfully regardless of part size
+
+## Export operation failure handling
+
+### RQ.ClickHouse.ExportPart.FailureHandling
+version: 1.0
+
+[ClickHouse] SHALL handle export operation failures in the following ways:
+* **Stateless Operation**: Export operations are stateless and ephemeral
+* **No Recovery**: If an export fails, it fails completely with no retry mechanism
+* **No State Persistence**: No export manifests or state are preserved across server restarts
+* **Simple Failure**: Export operations either succeed completely or fail with an error message
+* **No Partial Exports**: Failed exports leave no partial or corrupted data in destination storage
+
+## Network resilience
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.PacketIssues
+version: 1.0
+
+[ClickHouse] SHALL handle network packet issues during export operations by:
+* Tolerating packet delay without data corruption or loss
+* Handling packet loss and retransmitting data as needed
+* Detecting and handling packet corruption to ensure data integrity
+* Managing packet duplication without data duplication in destination
+* Handling packet reordering to maintain correct data sequence
+* Operating correctly under packet rate limiting constraints
+* Completing exports successfully despite network impairments
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.DestinationInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle destination storage interruptions during export operations by:
+* Detecting when destination storage becomes unavailable during export
+* Failing export operations gracefully when destination storage is unavailable
+* Logging failed exports in the `system.events` table with `PartsExportFailures` counter
+* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability
+* Allowing exports to complete successfully once destination storage becomes available again
+
+### RQ.ClickHouse.ExportPart.NetworkResilience.NodeInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:
+* Handling node restarts gracefully during export operations
+* Not leaving partial or corrupted data in destination storage when node restarts occur
+* With safe shutdown, ensuring exports complete successfully before node shutdown
+* With unsafe shutdown, allowing partial exports to complete successfully after node restart
+* Maintaining data integrity in destination storage regardless of node interruption type
+
+## Export operation restrictions
+
+### Preventing same table exports
+
+#### RQ.ClickHouse.ExportPart.Restrictions.SameTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting parts to the same table as the source by:
+* Validating that source and destination table identifiers are different
+* Throwing a `BAD_ARGUMENTS` exception with message "Exporting to the same table is not allowed" when source and destination are identical
+* Performing this validation before any export processing begins
+
+### Destination table compatibility
+
+#### RQ.ClickHouse.ExportPart.Restrictions.DestinationSupport
+version: 1.0
+
+[ClickHouse] SHALL validate destination table compatibility by:
+
+* Checking that the destination storage supports importing MergeTree parts
+* Verifying that the destination uses Hive partitioning strategy (`partition_strategy = 'hive'`)
+* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met
+* Performing this validation during the initial export setup phase
+
+### Local table restriction
+
+#### RQ.ClickHouse.ExportPart.Restrictions.LocalTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting parts to local MergeTree tables by:
+* Rejecting export operations where the destination table uses a MergeTree engine
+* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table
+* Performing this validation during the initial export setup phase
+
+### Partition key compatibility
+
+#### RQ.ClickHouse.ExportPart.Restrictions.PartitionKey
+version: 1.0
+
+[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:
+* Checking that the partition key expression matches between source and destination tables
+* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ
+* Performing this validation during the initial export setup phase
+
+### Source part availability
+
+#### RQ.ClickHouse.ExportPart.Restrictions.SourcePart
+version: 1.0
+
+[ClickHouse] SHALL validate source part availability by:
+* Checking that the specified part exists in the source table
+* Verifying the part is in an active state (not detached or missing)
+* Throwing an exception with message containing "Unexpected part name" when the part is not found
+* Performing this validation before creating the export manifest
+
+## Export operation concurrency
+
+### RQ.ClickHouse.ExportPart.Concurrency
+version: 1.0
+
+[ClickHouse] SHALL support concurrent export operations by:
+
+* Allowing multiple exports to run simultaneously without interference
+* Processing export operations asynchronously in the background
+* Preventing race conditions and data corruption during concurrent operations
+* Supporting concurrent exports of different parts to different destinations
+* Preventing concurrent exports of the same part to the same destination
+* Maintaining separate progress tracking and state for each concurrent operation
+* Ensuring thread safety across all concurrent export operations
+
+## Export operation idempotency
+
+### RQ.ClickHouse.ExportPart.Idempotency
+version: 1.0
+
+[ClickHouse] SHALL handle duplicate export operations by:
+* Preventing duplicate data from being exported when the same part is exported multiple times to the same destination
+* Detecting when an export operation attempts to export a part that already exists in the destination
+* Logging duplicate export attempts in the `system.events` table with the `PartsExportDuplicated` counter
+* Ensuring that destination data matches source data without duplication when the same part is exported multiple times
+
+## Export operation logging
+
+### RQ.ClickHouse.ExportPart.Logging
+version: 1.0
+
+[ClickHouse] SHALL provide detailed logging for export operations by:
+* Logging all export operations (both successful and failed) with timestamps and details
+* Recording the specific part name in the `system.part_log` table for all operations
+* Logging export events in the `system.events` table, including:
+ * `PartsExports` - Number of successful part exports
+ * `PartsExportFailures` - Number of failed part exports
+ * `PartsExportDuplicated` - Number of part exports that failed because target already exists
+* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PART`
+* Providing sufficient detail for monitoring and troubleshooting export operations
+
+## Monitoring export operations
+
+### RQ.ClickHouse.ExportPart.SystemTables.Exports
+version: 1.0
+
+[ClickHouse] SHALL provide a `system.exports` table that allows users to monitor active export operations with at least the following columns:
+* `source_table` - source table identifier
+* `destination_table` - destination table identifier
+
+The table SHALL track export operations before they complete and SHALL be empty after all exports complete.
+
+## Enabling export functionality
+
+### RQ.ClickHouse.ExportPart.Settings.AllowExperimental
+version: 1.0
+
+[ClickHouse] SHALL support the `allow_experimental_export_merge_tree_part` setting that SHALL gate the experimental export part functionality, which SHALL be set to `1` to enable `ALTER TABLE ... EXPORT PART ...` commands. The default value SHALL be `0` (turned off).
+
+## Handling file conflicts during export
+
+### RQ.ClickHouse.ExportPart.Settings.OverwriteFile
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_part_overwrite_file_if_exists` setting that controls whether to overwrite files if they already exist when exporting a merge tree part. The default value SHALL be `0` (turned off).
+
+## Export operation configuration
+
+### RQ.ClickHouse.ExportPart.ParallelFormatting
+version: 1.0
+
+[ClickHouse] SHALL support parallel formatting for export operations by:
+* Automatically enabling parallel formatting for large export operations to improve performance
+* Using the `output_format_parallel_formatting` setting to control parallel formatting behavior
+* Optimizing data processing based on export size and system resources
+* Providing consistent formatting performance across different export scenarios
+
+## Controlling export performance
+
+### RQ.ClickHouse.ExportPart.ServerSettings.MaxBandwidth
+version: 1.0
+
+[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.
+
+### RQ.ClickHouse.ExportPart.ServerSettings.BackgroundMovePoolSize
+version: 1.0
+
+[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.
+
+### RQ.ClickHouse.ExportPart.Metrics.Export
+version: 1.0
+
+[ClickHouse] SHALL provide the `Export` current metric in `system.metrics` table that tracks the number of currently executing exports.
+
+## Export operation security
+
+### RQ.ClickHouse.ExportPart.Security
+version: 1.0
+
+[ClickHouse] SHALL enforce security requirements for export operations:
+* **RBAC**: Users must have the following privileges:
+ * **Source Table**: `SELECT` privilege on the source table to read data parts
+ * **Destination Table**: `INSERT` privilege on the destination table to write exported data
+ * **Database Access**: `SHOW` privilege on both source and destination databases
+ * **System Tables**: `SELECT` privilege on `system.tables` to validate table existence
+* **Data Encryption**: All data in transit to destination storage must be encrypted using TLS/SSL
+* **Network Security**: Export operations must use secure connections to destination storage (HTTPS for S3, secure protocols for other storage)
+* **Credential Management**: Export operations must use secure credential storage and avoid exposing credentials in logs
+
+
+[ClickHouse]: https://clickhouse.com
+""",
+)
diff --git a/s3/requirements/export_partition.md b/s3/requirements/export_partition.md
new file mode 100644
index 000000000..2ea943237
--- /dev/null
+++ b/s3/requirements/export_partition.md
@@ -0,0 +1,790 @@
+# SRS-016 ClickHouse Export Partition to S3
+# Software Requirements Specification
+
+## Table of Contents
+
+* 1 [Introduction](#introduction)
+* 2 [Exporting Partitions to S3](#exporting-partitions-to-s3)
+ * 2.1 [RQ.ClickHouse.ExportPartition.S3](#rqclickhouseexportpartitions3)
+ * 2.2 [RQ.ClickHouse.ExportPartition.EmptyPartition](#rqclickhouseexportpartitionemptypartition)
+* 3 [SQL command support](#sql-command-support)
+ * 3.1 [RQ.ClickHouse.ExportPartition.SQLCommand](#rqclickhouseexportpartitionsqlcommand)
+ * 3.2 [RQ.ClickHouse.ExportPartition.IntoOutfile](#rqclickhouseexportpartitionintooutfile)
+ * 3.3 [RQ.ClickHouse.ExportPartition.Format](#rqclickhouseexportpartitionformat)
+ * 3.4 [RQ.ClickHouse.ExportPartition.SettingsClause](#rqclickhouseexportpartitionsettingsclause)
+* 4 [Supported source table engines](#supported-source-table-engines)
+ * 4.1 [RQ.ClickHouse.ExportPartition.SourceEngines](#rqclickhouseexportpartitionsourceengines)
+* 5 [Cluster and node support](#cluster-and-node-support)
+ * 5.1 [RQ.ClickHouse.ExportPartition.ClustersNodes](#rqclickhouseexportpartitionclustersnodes)
+ * 5.2 [RQ.ClickHouse.ExportPartition.Shards](#rqclickhouseexportpartitionshards)
+ * 5.3 [RQ.ClickHouse.ExportPartition.Versions](#rqclickhouseexportpartitionversions)
+* 6 [Supported source part storage types](#supported-source-part-storage-types)
+ * 6.1 [RQ.ClickHouse.ExportPartition.SourcePartStorage](#rqclickhouseexportpartitionsourcepartstorage)
+* 7 [Storage policies and volumes](#storage-policies-and-volumes)
+ * 7.1 [RQ.ClickHouse.ExportPartition.StoragePolicies](#rqclickhouseexportpartitionstoragepolicies)
+* 8 [Supported destination table engines](#supported-destination-table-engines)
+ * 8.1 [RQ.ClickHouse.ExportPartition.DestinationEngines](#rqclickhouseexportpartitiondestinationengines)
+* 9 [Temporary tables](#temporary-tables)
+ * 9.1 [RQ.ClickHouse.ExportPartition.TemporaryTable](#rqclickhouseexportpartitiontemporarytable)
+* 10 [Schema compatibility](#schema-compatibility)
+ * 10.1 [RQ.ClickHouse.ExportPartition.SchemaCompatibility](#rqclickhouseexportpartitionschemacompatibility)
+* 11 [Partition key types support](#partition-key-types-support)
+ * 11.1 [RQ.ClickHouse.ExportPartition.PartitionKeyTypes](#rqclickhouseexportpartitionpartitionkeytypes)
+* 12 [Partition content support](#partition-content-support)
+ * 12.1 [RQ.ClickHouse.ExportPartition.PartitionContent](#rqclickhouseexportpartitionpartitioncontent)
+ * 12.2 [RQ.ClickHouse.ExportPartition.SchemaChangeIsolation](#rqclickhouseexportpartitionschemachangeisolation)
+ * 12.3 [RQ.ClickHouse.ExportPartition.LargePartitions](#rqclickhouseexportpartitionlargepartitions)
+ * 12.4 [RQ.ClickHouse.ExportPartition.Corrupted](#rqclickhouseexportpartitioncorrupted)
+* 13 [Export operation failure handling](#export-operation-failure-handling)
+ * 13.1 [RQ.ClickHouse.ExportPartition.RetryMechanism](#rqclickhouseexportpartitionretrymechanism)
+ * 13.2 [RQ.ClickHouse.ExportPartition.Settings.MaxRetries](#rqclickhouseexportpartitionsettingsmaxretries)
+ * 13.3 [RQ.ClickHouse.ExportPartition.ResumeAfterFailure](#rqclickhouseexportpartitionresumeafterfailure)
+ * 13.4 [RQ.ClickHouse.ExportPartition.PartialProgress](#rqclickhouseexportpartitionpartialprogress)
+ * 13.5 [RQ.ClickHouse.ExportPartition.Cleanup](#rqclickhouseexportpartitioncleanup)
+ * 13.6 [RQ.ClickHouse.ExportPartition.Settings.ManifestTTL](#rqclickhouseexportpartitionsettingsmanifestttl)
+* 14 [Network resilience](#network-resilience)
+ * 14.1 [RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues](#rqclickhouseexportpartitionnetworkresiliencepacketissues)
+ * 14.2 [RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption](#rqclickhouseexportpartitionnetworkresiliencedestinationinterruption)
+ * 14.3 [RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption](#rqclickhouseexportpartitionnetworkresiliencenodeinterruption)
+* 15 [Export operation restrictions](#export-operation-restrictions)
+ * 15.1 [Preventing same table exports](#preventing-same-table-exports)
+ * 15.1.1 [RQ.ClickHouse.ExportPartition.Restrictions.SameTable](#rqclickhouseexportpartitionrestrictionssametable)
+ * 15.2 [Destination table compatibility](#destination-table-compatibility)
+ * 15.2.1 [RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport](#rqclickhouseexportpartitionrestrictionsdestinationsupport)
+ * 15.3 [Local table restriction](#local-table-restriction)
+ * 15.3.1 [RQ.ClickHouse.ExportPartition.Restrictions.LocalTable](#rqclickhouseexportpartitionrestrictionslocaltable)
+ * 15.4 [Partition key compatibility](#partition-key-compatibility)
+ * 15.4.1 [RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey](#rqclickhouseexportpartitionrestrictionspartitionkey)
+ * 15.5 [Source partition availability](#source-partition-availability)
+ * 15.5.1 [RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition](#rqclickhouseexportpartitionrestrictionssourcepartition)
+* 16 [Export operation concurrency](#export-operation-concurrency)
+ * 16.1 [RQ.ClickHouse.ExportPartition.Concurrency](#rqclickhouseexportpartitionconcurrency)
+* 17 [Export operation idempotency](#export-operation-idempotency)
+ * 17.1 [RQ.ClickHouse.ExportPartition.Idempotency](#rqclickhouseexportpartitionidempotency)
+ * 17.2 [RQ.ClickHouse.ExportPartition.Settings.ForceExport](#rqclickhouseexportpartitionsettingsforceexport)
+* 18 [Export operation logging](#export-operation-logging)
+ * 18.1 [RQ.ClickHouse.ExportPartition.Logging](#rqclickhouseexportpartitionlogging)
+* 19 [Monitoring export operations](#monitoring-export-operations)
+ * 19.1 [RQ.ClickHouse.ExportPartition.SystemTables.Exports](#rqclickhouseexportpartitionsystemtablesexports)
+* 20 [Enabling export functionality](#enabling-export-functionality)
+ * 20.1 [RQ.ClickHouse.ExportPartition.Settings.AllowExperimental](#rqclickhouseexportpartitionsettingsallowexperimental)
+ * 20.2 [RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled](#rqclickhouseexportpartitionsettingsallowexperimentaldisabled)
+* 21 [Handling file conflicts during export](#handling-file-conflicts-during-export)
+ * 21.1 [RQ.ClickHouse.ExportPartition.Settings.OverwriteFile](#rqclickhouseexportpartitionsettingsoverwritefile)
+* 22 [Export operation configuration](#export-operation-configuration)
+ * 22.1 [RQ.ClickHouse.ExportPartition.ParallelFormatting](#rqclickhouseexportpartitionparallelformatting)
+* 23 [Controlling export performance](#controlling-export-performance)
+ * 23.1 [RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth](#rqclickhouseexportpartitionserversettingsmaxbandwidth)
+ * 23.2 [RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize](#rqclickhouseexportpartitionserversettingsbackgroundmovepoolsize)
+ * 23.3 [RQ.ClickHouse.ExportPartition.Metrics.Export](#rqclickhouseexportpartitionmetricsexport)
+* 24 [Export operation security](#export-operation-security)
+ * 24.1 [RQ.ClickHouse.ExportPartition.Security.RBAC](#rqclickhouseexportpartitionsecurityrbac)
+ * 24.2 [RQ.ClickHouse.ExportPartition.Security.DataEncryption](#rqclickhouseexportpartitionsecuritydataencryption)
+ * 24.3 [RQ.ClickHouse.ExportPartition.Security.Network](#rqclickhouseexportpartitionsecuritynetwork)
+ * 24.4 [RQ.ClickHouse.ExportPartition.Security.CredentialManagement](#rqclickhouseexportpartitionsecuritycredentialmanagement)
+
+## Introduction
+
+This specification defines requirements for exporting partitions (all parts within a partition) from ReplicatedMergeTree tables to S3-compatible object storage. This feature enables users to export entire partitions containing multiple data parts across cluster nodes.
+
+## Exporting Partitions to S3
+
+### RQ.ClickHouse.ExportPartition.S3
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions (all parts within a partition) from ReplicatedMergeTree engine tables to S3 object storage. The export operation SHALL export all parts that belong to the specified partition ID, ensuring complete partition data is transferred to the destination.
+
+### RQ.ClickHouse.ExportPartition.EmptyPartition
+version: 1.0
+
+[ClickHouse] SHALL support exporting from empty partitions by:
+* Completing export operations successfully when the specified partition contains no parts
+* Resulting in an empty destination partition when exporting from an empty source partition
+* Not creating any files in destination storage when there are no parts to export in the partition
+* Handling empty partitions gracefully without errors
+
+## SQL command support
+
+### RQ.ClickHouse.ExportPartition.SQLCommand
+version: 1.0
+
+[ClickHouse] SHALL support the following SQL command syntax for exporting partitions from ReplicatedMergeTree tables to object storage tables:
+
+```sql
+ALTER TABLE [database.]source_table_name
+EXPORT PARTITION ID 'partition_id'
+TO TABLE [database.]destination_table_name
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+**Parameters:**
+- `source_table_name`: Name of the source ReplicatedMergeTree table
+- `partition_id`: The partition ID to export (string literal), which identifies all parts belonging to that partition
+- `destination_table_name`: Name of the destination object storage table
+- `allow_experimental_export_merge_tree_part`: Setting that must be set to `1` to enable this experimental feature
+
+This command allows users to export entire partitions in a single operation, which is more efficient than exporting individual parts and ensures all data for a partition is exported together.
+
+### RQ.ClickHouse.ExportPartition.IntoOutfile
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `INTO OUTFILE` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+INTO OUTFILE '/path/to/file'
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### RQ.ClickHouse.ExportPartition.Format
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `FORMAT` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+FORMAT JSON
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### RQ.ClickHouse.ExportPartition.SettingsClause
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `SETTINGS` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_max_retries = 5
+```
+
+## Supported source table engines
+
+### RQ.ClickHouse.ExportPartition.SourceEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from the following source table engines:
+* `ReplicatedMergeTree` - Replicated MergeTree engine (primary use case)
+* `ReplicatedSummingMergeTree` - Replicated MergeTree with automatic summation
+* `ReplicatedAggregatingMergeTree` - Replicated MergeTree with pre-aggregated data
+* `ReplicatedCollapsingMergeTree` - Replicated MergeTree with row versioning
+* `ReplicatedVersionedCollapsingMergeTree` - Replicated CollapsingMergeTree with version tracking
+* `ReplicatedGraphiteMergeTree` - Replicated MergeTree optimized for Graphite data
+* All other ReplicatedMergeTree family engines
+
+Export partition functionality manages export operations across multiple replicas in a cluster, ensuring that parts are exported correctly and avoiding conflicts.
+
+## Cluster and node support
+
+### RQ.ClickHouse.ExportPartition.ClustersNodes
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from multiple nodes in a ReplicatedMergeTree cluster to the same destination storage, ensuring that:
+* Each replica in the cluster can independently export parts from the partition that it owns locally
+* All parts within a partition are exported exactly once, even when distributed across multiple replicas
+* Exported data from different replicas is correctly aggregated in the destination storage
+* All nodes in the cluster can read the same exported partition data from the destination
+* Export operations continue to make progress even if some replicas are temporarily unavailable
+
+In a replicated cluster, different parts of the same partition may exist on different replicas. The system must coordinate exports across all replicas to ensure complete partition export without duplication.
+
+### RQ.ClickHouse.ExportPartition.Shards
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from source tables that are on different shards than the destination table.
+
+### RQ.ClickHouse.ExportPartition.Versions
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from source tables that are stored on servers with different ClickHouse versions than the destination server.
+
+Users can export partitions from tables on servers with older ClickHouse versions to tables on servers with newer versions, enabling data migration and version upgrades.
+
+## Supported source part storage types
+
+### RQ.ClickHouse.ExportPartition.SourcePartStorage
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions regardless of the underlying storage type where the source parts are stored, including:
+* **Local Disks**: Parts stored on local filesystem
+* **S3/Object Storage**: Parts stored on S3 or S3-compatible object storage
+* **Encrypted Disks**: Parts stored on encrypted disks (disk-level encryption)
+* **Cached Disks**: Parts stored with filesystem cache enabled
+* **Remote Disks**: Parts stored on HDFS, Azure Blob Storage, or Google Cloud Storage
+* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)
+* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled
+
+Users should be able to export partitions regardless of where the source data is physically stored, providing flexibility in storage configurations.
+
+## Storage policies and volumes
+
+### RQ.ClickHouse.ExportPartition.StoragePolicies
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:
+* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks
+* **External Volumes**: Volumes using external storage systems
+* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers
+* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks
+* Exporting all parts in a partition regardless of which volume or disk within the storage policy contains each part
+* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy
+
+Users may have partitions with parts distributed across different storage tiers or volumes, and the export should handle all parts regardless of their storage location.
+
+## Supported destination table engines
+
+### RQ.ClickHouse.ExportPartition.DestinationEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting to destination tables that:
+* Support object storage engines including:
+ * `S3` - Amazon S3 and S3-compatible storage
+ * `StorageObjectStorage` - Generic object storage interface
+ * `HDFS` - Hadoop Distributed File System (with Hive partitioning)
+ * `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)
+ * `GCS` - Google Cloud Storage (with Hive partitioning)
+
+Export partition is designed to move data from local or replicated storage to object storage systems for long-term storage, analytics, or data sharing purposes.
+
+## Temporary tables
+
+### RQ.ClickHouse.ExportPartition.TemporaryTable
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from temporary ReplicatedMergeTree tables to destination object storage tables.
+
+For example,
+
+```sql
+CREATE TEMPORARY TABLE temp_table (p UInt64, k String, d UInt64)
+ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/temp_table', '{replica}')
+PARTITION BY p ORDER BY k;
+
+INSERT INTO temp_table VALUES (2020, 'key1', 100), (2020, 'key2', 200);
+
+ALTER TABLE temp_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+## Schema compatibility
+
+### RQ.ClickHouse.ExportPartition.SchemaCompatibility
+version: 1.0
+
+[ClickHouse] SHALL require source and destination tables to have compatible schemas for successful export operations:
+* Identical physical column schemas between source and destination
+* The same partition key expression in both tables
+* Compatible data types for all columns
+* Matching column order and names
+
+Schema compatibility ensures that exported data can be correctly read from the destination table without data loss or corruption.
+
+## Partition key types support
+
+### RQ.ClickHouse.ExportPartition.PartitionKeyTypes
+version: 1.0
+
+[ClickHouse] SHALL support export operations for tables with partition key types that are compatible with Hive partitioning, as shown in the following table:
+
+| Partition Key Type | Supported | Examples | Notes |
+|-------------------------|-----------|--------------------------------------------------------------------------|--------------------------------|
+| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |
+| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |
+| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |
+| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |
+
+[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.
+
+[ClickHouse] SHALL require destination tables to support Hive partitioning, which limits the supported partition key types to Integer, Date/DateTime, and String types. Complex expressions that result in unsupported types are not supported for export operations.
+
+Hive partitioning is a standard way to organize data in object storage systems, making exported data compatible with various analytics tools and systems.
+
+## Partition content support
+
+### RQ.ClickHouse.ExportPartition.PartitionContent
+version: 1.0
+
+[ClickHouse] SHALL support export operations for partitions containing all valid MergeTree part types and their contents, including:
+
+| Part Type | Supported | Description | Special Features |
+|-------------------|-----------|--------------------------------------------------------------|--------------------------------|
+| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |
+| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |
+
+[ClickHouse] SHALL export all parts within the specified partition, regardless of their type. The system SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL maintain data integrity in the destination storage.
+
+Partitions may contain a mix of different part types, and the export must handle all of them correctly to ensure complete partition export.
+
+### RQ.ClickHouse.ExportPartition.SchemaChangeIsolation
+version: 1.0
+
+[ClickHouse] SHALL ensure exported partition data is isolated from subsequent schema changes by:
+* Preserving exported data exactly as it was at the time of export
+* Not being affected by schema changes (column drops, renames, type changes) that occur after export
+* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export
+* Ensuring exported data reflects the source table state at the time of export, not the current state
+
+Once a partition is exported, the exported data should remain stable and not be affected by future changes to the source table schema.
+
+### RQ.ClickHouse.ExportPartition.LargePartitions
+version: 1.0
+
+[ClickHouse] SHALL support exporting large partitions by:
+* Handling partitions with large numbers of parts (e.g., hundreds or thousands of parts)
+* Processing partitions with large numbers of rows (e.g., billions of rows)
+* Processing large data volumes efficiently during export
+* Maintaining data integrity when exporting large partitions
+* Completing export operations successfully regardless of partition size
+* Allowing export operations to continue over extended periods of time for very large partitions
+
+Production systems often have partitions containing very large amounts of data, and the export must handle these efficiently without timeouts or memory issues.
+
+### RQ.ClickHouse.ExportPartition.Corrupted
+version: 1.0
+
+[ClickHouse] SHALL output an error and prevent export operations from proceeding when trying to export a partition that contains corrupted parts in the source table.
+
+The system SHALL detect corruption in partitions containing compact parts, wide parts, or mixed part types.
+
+## Export operation failure handling
+
+### RQ.ClickHouse.ExportPartition.RetryMechanism
+version: 1.0
+
+[ClickHouse] SHALL automatically retry failed part exports within a partition up to a configurable maximum retry count. If all retry attempts are exhausted for a part, the entire partition export operation SHALL be marked as failed.
+
+Unlike single-part exports, partition exports involve multiple parts and may take significant time. Retry mechanisms ensure that temporary failures don't require restarting the entire export operation.
+
+### RQ.ClickHouse.ExportPartition.Settings.MaxRetries
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_max_retries` setting that controls the maximum number of retries for exporting a merge tree part in an export partition task. The default value SHALL be `3`.
+
+This setting allows users to control how many times the system will retry exporting a part before marking it as failed.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_max_retries = 5
+```
+
+### RQ.ClickHouse.ExportPartition.ResumeAfterFailure
+version: 1.0
+
+[ClickHouse] SHALL allow export operations to resume after node failures or restarts. The system SHALL track which parts have been successfully exported and SHALL not re-export parts that were already successfully exported.
+
+### RQ.ClickHouse.ExportPartition.PartialProgress
+version: 1.0
+
+[ClickHouse] SHALL allow export operations to make partial progress, with successfully exported parts remaining in the destination even if other parts fail. Users SHALL be able to see which parts have been successfully exported and which parts have failed.
+
+For example, users can query the export status to see partial progress:
+
+```sql
+SELECT source_table, destination_table, partition_id, status,
+ parts_total, parts_processed, parts_failed
+FROM system.replicated_partition_exports
+WHERE partition_id = '2020'
+```
+
+### RQ.ClickHouse.ExportPartition.Cleanup
+version: 1.0
+
+[ClickHouse] SHALL automatically clean up failed or completed export operations after a configurable TTL period.
+
+### RQ.ClickHouse.ExportPartition.Settings.ManifestTTL
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_manifest_ttl` setting that determines how long the export manifest will be retained. This setting prevents the same partition from being exported twice to the same destination within the TTL period. The default value SHALL be `180` seconds.
+
+This setting only affects completed export operations and does not delete in-progress tasks. It allows users to control how long export history is maintained to prevent duplicate exports.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_manifest_ttl = 360
+```
+
+## Network resilience
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues
+version: 1.0
+
+[ClickHouse] SHALL handle network packet issues during export operations by:
+* Tolerating packet delay without data corruption or loss
+* Handling packet loss and retransmitting data as needed
+* Detecting and handling packet corruption to ensure data integrity
+* Managing packet duplication without data duplication in destination
+* Handling packet reordering to maintain correct data sequence
+* Operating correctly under packet rate limiting constraints
+* Completing exports successfully despite network impairments
+
+Network issues are common in distributed systems, and export operations must be resilient to ensure data integrity.
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle destination storage interruptions during export operations by:
+* Detecting when destination storage becomes unavailable during export
+* Retrying failed part exports when destination storage becomes available again
+* Logging failed exports in the `system.events` table with appropriate counters
+* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability
+* Allowing exports to complete successfully once destination storage becomes available again
+* Resuming export operations from the last successfully exported part
+
+Destination storage systems may experience temporary outages, and the export should automatically recover when service is restored.
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:
+* Allowing export operations to resume after node restart without data loss or duplication
+* Allowing other replicas to continue or complete export operations if a node fails
+* Not leaving partial or corrupted data in destination storage when node restarts occur
+* With safe shutdown, ensuring exports complete successfully before node shutdown when possible
+* With unsafe shutdown, allowing export operations to resume from the last checkpoint after node restart
+* Maintaining data integrity in destination storage regardless of node interruption type
+* Ensuring that parts already exported are not re-exported after node restart
+
+Node failures are common in distributed systems, and export operations must be able to recover and continue without data loss or duplication.
+
+## Export operation restrictions
+
+### Preventing same table exports
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.SameTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting partitions to the same table as the source by:
+* Validating that source and destination table identifiers are different
+* Throwing a `BAD_ARGUMENTS` exception with message "Exporting to the same table is not allowed" when source and destination are identical
+* Performing this validation before any export processing begins
+
+Exporting to the same table would be redundant and could cause data duplication or conflicts.
+
+For example, the following command SHALL output an error:
+
+```sql
+ALTER TABLE my_table
+EXPORT PARTITION ID '2020'
+TO TABLE my_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Destination table compatibility
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport
+version: 1.0
+
+[ClickHouse] SHALL validate destination table compatibility by:
+
+* Checking that the destination storage supports importing MergeTree parts
+* Verifying that the destination uses Hive partitioning strategy (`partition_strategy = 'hive'`)
+* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met
+* Performing this validation during the initial export setup phase
+
+The destination must support the format and partitioning strategy required for exported data.
+
+### Local table restriction
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.LocalTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting partitions to local MergeTree tables by:
+* Rejecting export operations where the destination table uses a MergeTree engine
+* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table
+* Performing this validation during the initial export setup phase
+
+Export partition is designed to move data to object storage, not to local MergeTree tables.
+
+For example, if `local_table` is a MergeTree table, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE local_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Partition key compatibility
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey
+version: 1.0
+
+[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:
+* Checking that the partition key expression matches between source and destination tables
+* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ
+* Performing this validation during the initial export setup phase
+
+Matching partition keys ensure that exported data is organized correctly in the destination storage.
+
+For example, if `source_table` is partitioned by `toYYYYMM(date)` and `destination_table` is partitioned by `toYYYYMMDD(date)`, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Source partition availability
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition
+version: 1.0
+
+[ClickHouse] SHALL validate source partition availability by:
+* Checking that the specified partition ID exists in the source table
+* Verifying that the partition contains at least one active part (not detached or missing)
+* Throwing an exception with an appropriate error message when the partition is not found or is empty
+* Performing this validation before any export processing begins
+
+The system must verify that the partition exists and contains data before attempting to export it.
+
+For example, if partition ID '2025' does not exist in `source_table`, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2025'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+## Export operation concurrency
+
+### RQ.ClickHouse.ExportPartition.Concurrency
+version: 1.0
+
+[ClickHouse] SHALL support concurrent export operations by:
+* Allowing multiple partition exports to run simultaneously without interference
+* Supporting concurrent exports of different partitions to different destinations
+* Preventing concurrent exports of the same partition to the same destination
+* Allowing different replicas to export different parts of the same partition concurrently
+* Maintaining separate progress tracking for each concurrent operation
+
+Multiple users may want to export different partitions simultaneously, and the system must coordinate these operations to prevent conflicts while maximizing parallelism.
+
+## Export operation idempotency
+
+### RQ.ClickHouse.ExportPartition.Idempotency
+version: 1.0
+
+[ClickHouse] SHALL handle duplicate export operations by:
+* Preventing duplicate data from being exported when the same partition is exported multiple times to the same destination
+* Detecting when a partition export is already in progress or completed
+* Detecting when an export operation attempts to export a partition that already exists in the destination
+* Logging duplicate export attempts in the `system.events` table with appropriate counters
+* Ensuring that destination data matches source data without duplication when the same partition is exported multiple times
+* Allowing users to force re-export of a partition if needed (e.g., after TTL expiration or manual cleanup)
+
+Users may accidentally trigger the same export multiple times, and the system should prevent duplicate data while allowing legitimate re-exports when needed.
+
+### RQ.ClickHouse.ExportPartition.Settings.ForceExport
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_force_export` setting that allows users to ignore existing partition export entries and force a new export operation. The default value SHALL be `false` (turned off).
+
+When set to `true`, this setting allows users to overwrite existing export entries and force re-export of a partition, even if a previous export operation exists for the same partition and destination.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_force_export = 1
+```
+
+## Export operation logging
+
+### RQ.ClickHouse.ExportPartition.Logging
+version: 1.0
+
+[ClickHouse] SHALL provide detailed logging for export operations by:
+* Logging all export operations (both successful and failed) with timestamps and details
+* Recording the specific partition ID in the `system.part_log` table for all operations
+* Logging export events in the `system.events` table, including:
+ * `PartsExports` - Number of successful part exports (within partitions)
+ * `PartsExportFailures` - Number of failed part exports
+ * `PartsExportDuplicated` - Number of part exports that failed because target already exists
+* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PARTITION`
+* Providing sufficient detail for monitoring and troubleshooting export operations
+* Logging per-part export status within partition exports
+
+Detailed logging helps users monitor export progress, troubleshoot issues, and audit export operations.
+
+For example, users can query export logs:
+
+```sql
+SELECT event_time, event_type, table, partition, rows, bytes_read, bytes_written
+FROM system.part_log
+WHERE event_type = 'EXPORT_PARTITION'
+ORDER BY event_time DESC
+```
+
+## Monitoring export operations
+
+### RQ.ClickHouse.ExportPartition.SystemTables.Exports
+version: 1.0
+
+[ClickHouse] SHALL provide a `system.replicated_partition_exports` table that allows users to monitor active partition export operations with at least the following columns:
+* `source_table` - source table identifier
+* `destination_table` - destination table identifier
+* `partition_id` - the partition ID being exported
+* `status` - current status of the export operation (e.g., PENDING, IN_PROGRESS, COMPLETED, FAILED)
+* `parts_total` - total number of parts in the partition
+* `parts_processed` - number of parts successfully exported
+* `parts_failed` - number of parts that failed to export
+* `create_time` - when the export operation was created
+* `update_time` - last update time of the export operation
+
+The table SHALL track export operations before they complete and SHALL show completed or failed exports until they are cleaned up (based on TTL).
+
+Users need visibility into export operations to monitor progress, identify issues, and understand export status across the cluster.
+
+For example,
+
+```sql
+SELECT source_table, destination_table, partition_id, status,
+ parts_total, parts_processed, parts_failed, create_time, update_time
+FROM system.replicated_partition_exports
+WHERE status = 'IN_PROGRESS'
+```
+
+## Enabling export functionality
+
+### RQ.ClickHouse.ExportPartition.Settings.AllowExperimental
+version: 1.0
+
+[ClickHouse] SHALL support the `allow_experimental_export_merge_tree_part` setting that SHALL gate the experimental export partition functionality, which SHALL be set to `1` to enable `ALTER TABLE ... EXPORT PARTITION ID ...` commands. The default value SHALL be `0` (turned off).
+
+This setting allows administrators to control access to experimental functionality and ensures users are aware they are using a feature that may change.
+
+### RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled
+version: 1.0
+
+[ClickHouse] SHALL prevent export partition operations when `allow_experimental_export_merge_tree_part` is set to `0` (turned off). When the setting is `0`, attempting to execute `ALTER TABLE ... EXPORT PARTITION ID ...` commands SHALL result in an error indicating that the experimental feature is not enabled.
+
+For example, the following command SHALL output an error when the setting is `0`:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+```
+
+## Handling file conflicts during export
+
+### RQ.ClickHouse.ExportPartition.Settings.OverwriteFile
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_part_overwrite_file_if_exists` setting that controls whether to overwrite files if they already exist when exporting a partition. The default value SHALL be `0` (turned off), meaning exports will fail if files already exist in the destination.
+
+This setting allows users to control whether to overwrite existing data in the destination, providing safety by default while allowing overwrites when needed.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_part_overwrite_file_if_exists = 1
+```
+
+## Export operation configuration
+
+### RQ.ClickHouse.ExportPartition.ParallelFormatting
+version: 1.0
+
+[ClickHouse] SHALL support parallel formatting for export operations by:
+* Automatically enabling parallel formatting for large export operations to improve performance
+* Using the `output_format_parallel_formatting` setting to control parallel formatting behavior
+* Optimizing data processing based on export size and system resources
+* Providing consistent formatting performance across different export scenarios
+* Allowing parallel processing of multiple parts within a partition when possible
+
+Parallel formatting improves export performance, especially for large partitions with many parts.
+
+## Controlling export performance
+
+### RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth
+version: 1.0
+
+[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.
+
+Administrators need to control export bandwidth to avoid impacting other operations on the server.
+
+### RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize
+version: 1.0
+
+[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.
+
+This setting allows administrators to balance export performance with other system operations.
+
+### RQ.ClickHouse.ExportPartition.Metrics.Export
+version: 1.0
+
+[ClickHouse] SHALL provide the `Export` current metric in `system.metrics` table that tracks the number of currently executing partition exports.
+
+This metric helps monitor system load from export operations.
+
+## Export operation security
+
+### RQ.ClickHouse.ExportPartition.Security.RBAC
+version: 1.0
+
+[ClickHouse] SHALL enforce role-based access control (RBAC) for export operations. Users must have the following privileges to perform export operations:
+* **Source Table**: `SELECT` privilege on the source table to read data parts
+* **Destination Table**: `INSERT` privilege on the destination table to write exported data
+* **Database Access**: `SHOW` privilege on both source and destination databases
+* **System Tables**: `SELECT` privilege on `system.tables` and `system.replicated_partition_exports` to validate table existence and monitor exports
+
+Export operations move potentially sensitive data, and proper access controls ensure only authorized users can export partitions.
+
+### RQ.ClickHouse.ExportPartition.Security.DataEncryption
+version: 1.0
+
+[ClickHouse] SHALL encrypt all data in transit to destination storage using TLS/SSL during export operations.
+
+Data encryption protects sensitive information from being intercepted or accessed during transmission to destination storage.
+
+### RQ.ClickHouse.ExportPartition.Security.Network
+version: 1.0
+
+[ClickHouse] SHALL use secure connections to destination storage during export operations. For S3-compatible storage, connections must use HTTPS. For other storage types, secure protocols appropriate to the storage system must be used.
+
+Secure network connections prevent unauthorized access and ensure data integrity during export operations.
+
+### RQ.ClickHouse.ExportPartition.Security.CredentialManagement
+version: 1.0
+
+[ClickHouse] SHALL use secure credential storage for export operations and SHALL avoid exposing credentials in logs or error messages.
+
+Proper credential management prevents unauthorized access to destination storage systems and protects sensitive authentication information.
+
+
+[ClickHouse]: https://clickhouse.com
+
diff --git a/s3/requirements/export_partition.py b/s3/requirements/export_partition.py
new file mode 100644
index 000000000..1e37b51b1
--- /dev/null
+++ b/s3/requirements/export_partition.py
@@ -0,0 +1,2262 @@
+# These requirements were auto generated
+# from software requirements specification (SRS)
+# document by TestFlows v2.0.250110.1002922.
+# Do not edit by hand but re-generate instead
+# using 'tfs requirements generate' command.
+from testflows.core import Specification
+from testflows.core import Requirement
+
+Heading = Specification.Heading
+
+RQ_ClickHouse_ExportPartition_S3 = Requirement(
+ name="RQ.ClickHouse.ExportPartition.S3",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions (all parts within a partition) from ReplicatedMergeTree engine tables to S3 object storage. The export operation SHALL export all parts that belong to the specified partition ID, ensuring complete partition data is transferred to the destination.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="2.1",
+)
+
+RQ_ClickHouse_ExportPartition_EmptyPartition = Requirement(
+ name="RQ.ClickHouse.ExportPartition.EmptyPartition",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting from empty partitions by:\n"
+ "* Completing export operations successfully when the specified partition contains no parts\n"
+ "* Resulting in an empty destination partition when exporting from an empty source partition\n"
+ "* Not creating any files in destination storage when there are no parts to export in the partition\n"
+ "* Handling empty partitions gracefully without errors\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="2.2",
+)
+
+RQ_ClickHouse_ExportPartition_SQLCommand = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SQLCommand",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the following SQL command syntax for exporting partitions from ReplicatedMergeTree tables to object storage tables:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE [database.]source_table_name \n"
+ "EXPORT PARTITION ID 'partition_id' \n"
+ "TO TABLE [database.]destination_table_name\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ "**Parameters:**\n"
+ "- `source_table_name`: Name of the source ReplicatedMergeTree table\n"
+ "- `partition_id`: The partition ID to export (string literal), which identifies all parts belonging to that partition\n"
+ "- `destination_table_name`: Name of the destination object storage table\n"
+ "- `allow_experimental_export_merge_tree_part`: Setting that must be set to `1` to enable this experimental feature\n"
+ "\n"
+ "This command allows users to export entire partitions in a single operation, which is more efficient than exporting individual parts and ensures all data for a partition is exported together.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="3.1",
+)
+
+RQ_ClickHouse_ExportPartition_IntoOutfile = Requirement(
+ name="RQ.ClickHouse.ExportPartition.IntoOutfile",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the usage of the `INTO OUTFILE` clause with `EXPORT PARTITION` and SHALL not output any errors.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "INTO OUTFILE '/path/to/file'\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="3.2",
+)
+
+RQ_ClickHouse_ExportPartition_Format = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Format",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the usage of the `FORMAT` clause with `EXPORT PARTITION` and SHALL not output any errors.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "FORMAT JSON\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="3.3",
+)
+
+RQ_ClickHouse_ExportPartition_SettingsClause = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SettingsClause",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the usage of the `SETTINGS` clause with `EXPORT PARTITION` and SHALL not output any errors.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1, \n"
+ " export_merge_tree_partition_max_retries = 5\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="3.4",
+)
+
+RQ_ClickHouse_ExportPartition_SourceEngines = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SourceEngines",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from the following source table engines:\n"
+ "* `ReplicatedMergeTree` - Replicated MergeTree engine (primary use case)\n"
+ "* `ReplicatedSummingMergeTree` - Replicated MergeTree with automatic summation\n"
+ "* `ReplicatedAggregatingMergeTree` - Replicated MergeTree with pre-aggregated data\n"
+ "* `ReplicatedCollapsingMergeTree` - Replicated MergeTree with row versioning\n"
+ "* `ReplicatedVersionedCollapsingMergeTree` - Replicated CollapsingMergeTree with version tracking\n"
+ "* `ReplicatedGraphiteMergeTree` - Replicated MergeTree optimized for Graphite data\n"
+ "* All other ReplicatedMergeTree family engines\n"
+ "\n"
+ "Export partition functionality manages export operations across multiple replicas in a cluster, ensuring that parts are exported correctly and avoiding conflicts.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="4.1",
+)
+
+RQ_ClickHouse_ExportPartition_ClustersNodes = Requirement(
+ name="RQ.ClickHouse.ExportPartition.ClustersNodes",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from multiple nodes in a ReplicatedMergeTree cluster to the same destination storage, ensuring that:\n"
+ "* Each replica in the cluster can independently export parts from the partition that it owns locally\n"
+ "* All parts within a partition are exported exactly once, even when distributed across multiple replicas\n"
+ "* Exported data from different replicas is correctly aggregated in the destination storage\n"
+ "* All nodes in the cluster can read the same exported partition data from the destination\n"
+ "* Export operations continue to make progress even if some replicas are temporarily unavailable\n"
+ "\n"
+ "In a replicated cluster, different parts of the same partition may exist on different replicas. The system must coordinate exports across all replicas to ensure complete partition export without duplication.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="5.1",
+)
+
+RQ_ClickHouse_ExportPartition_Shards = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Shards",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from source tables that are on different shards than the destination table.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="5.2",
+)
+
+RQ_ClickHouse_ExportPartition_Versions = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Versions",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from source tables that are stored on servers with different ClickHouse versions than the destination server.\n"
+ "\n"
+ "Users can export partitions from tables on servers with older ClickHouse versions to tables on servers with newer versions, enabling data migration and version upgrades.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="5.3",
+)
+
+RQ_ClickHouse_ExportPartition_SourcePartStorage = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SourcePartStorage",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions regardless of the underlying storage type where the source parts are stored, including:\n"
+ "* **Local Disks**: Parts stored on local filesystem\n"
+ "* **S3/Object Storage**: Parts stored on S3 or S3-compatible object storage\n"
+ "* **Encrypted Disks**: Parts stored on encrypted disks (disk-level encryption)\n"
+ "* **Cached Disks**: Parts stored with filesystem cache enabled\n"
+ "* **Remote Disks**: Parts stored on HDFS, Azure Blob Storage, or Google Cloud Storage\n"
+ "* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)\n"
+ "* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled\n"
+ "\n"
+ "Users should be able to export partitions regardless of where the source data is physically stored, providing flexibility in storage configurations.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="6.1",
+)
+
+RQ_ClickHouse_ExportPartition_StoragePolicies = Requirement(
+ name="RQ.ClickHouse.ExportPartition.StoragePolicies",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:\n"
+ "* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks\n"
+ "* **External Volumes**: Volumes using external storage systems\n"
+ "* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers\n"
+ "* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks\n"
+ "* Exporting all parts in a partition regardless of which volume or disk within the storage policy contains each part\n"
+ "* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy\n"
+ "\n"
+ "Users may have partitions with parts distributed across different storage tiers or volumes, and the export should handle all parts regardless of their storage location.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="7.1",
+)
+
+RQ_ClickHouse_ExportPartition_DestinationEngines = Requirement(
+ name="RQ.ClickHouse.ExportPartition.DestinationEngines",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting to destination tables that:\n"
+ "* Support object storage engines including:\n"
+ " * `S3` - Amazon S3 and S3-compatible storage\n"
+ " * `StorageObjectStorage` - Generic object storage interface\n"
+ " * `HDFS` - Hadoop Distributed File System (with Hive partitioning)\n"
+ " * `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)\n"
+ " * `GCS` - Google Cloud Storage (with Hive partitioning)\n"
+ "\n"
+ "Export partition is designed to move data from local or replicated storage to object storage systems for long-term storage, analytics, or data sharing purposes.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="8.1",
+)
+
+RQ_ClickHouse_ExportPartition_TemporaryTable = Requirement(
+ name="RQ.ClickHouse.ExportPartition.TemporaryTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting partitions from temporary ReplicatedMergeTree tables to destination object storage tables.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "CREATE TEMPORARY TABLE temp_table (p UInt64, k String, d UInt64) \n"
+ "ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/temp_table', '{replica}') \n"
+ "PARTITION BY p ORDER BY k;\n"
+ "\n"
+ "INSERT INTO temp_table VALUES (2020, 'key1', 100), (2020, 'key2', 200);\n"
+ "\n"
+ "ALTER TABLE temp_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="9.1",
+)
+
+RQ_ClickHouse_ExportPartition_SchemaCompatibility = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SchemaCompatibility",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL require source and destination tables to have compatible schemas for successful export operations:\n"
+ "* Identical physical column schemas between source and destination\n"
+ "* The same partition key expression in both tables\n"
+ "* Compatible data types for all columns\n"
+ "* Matching column order and names\n"
+ "\n"
+ "Schema compatibility ensures that exported data can be correctly read from the destination table without data loss or corruption.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="10.1",
+)
+
+RQ_ClickHouse_ExportPartition_PartitionKeyTypes = Requirement(
+ name="RQ.ClickHouse.ExportPartition.PartitionKeyTypes",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support export operations for tables with partition key types that are compatible with Hive partitioning, as shown in the following table:\n"
+ "\n"
+ "| Partition Key Type | Supported | Examples | Notes |\n"
+ "|-------------------------|-----------|--------------------------------------------------------------------------|--------------------------------|\n"
+ "| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |\n"
+ "| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |\n"
+ "| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |\n"
+ "| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |\n"
+ "\n"
+ "[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.\n"
+ "\n"
+ "[ClickHouse] SHALL require destination tables to support Hive partitioning, which limits the supported partition key types to Integer, Date/DateTime, and String types. Complex expressions that result in unsupported types are not supported for export operations.\n"
+ "\n"
+ "Hive partitioning is a standard way to organize data in object storage systems, making exported data compatible with various analytics tools and systems.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="11.1",
+)
+
+RQ_ClickHouse_ExportPartition_PartitionContent = Requirement(
+ name="RQ.ClickHouse.ExportPartition.PartitionContent",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support export operations for partitions containing all valid MergeTree part types and their contents, including:\n"
+ "\n"
+ "| Part Type | Supported | Description | Special Features |\n"
+ "|-------------------|-----------|--------------------------------------------------------------|--------------------------------|\n"
+ "| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |\n"
+ "| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |\n"
+ "\n"
+ "[ClickHouse] SHALL export all parts within the specified partition, regardless of their type. The system SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL maintain data integrity in the destination storage.\n"
+ "\n"
+ "Partitions may contain a mix of different part types, and the export must handle all of them correctly to ensure complete partition export.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="12.1",
+)
+
+RQ_ClickHouse_ExportPartition_SchemaChangeIsolation = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SchemaChangeIsolation",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL ensure exported partition data is isolated from subsequent schema changes by:\n"
+ "* Preserving exported data exactly as it was at the time of export\n"
+ "* Not being affected by schema changes (column drops, renames, type changes) that occur after export\n"
+ "* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export\n"
+ "* Ensuring exported data reflects the source table state at the time of export, not the current state\n"
+ "\n"
+ "Once a partition is exported, the exported data should remain stable and not be affected by future changes to the source table schema.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="12.2",
+)
+
+RQ_ClickHouse_ExportPartition_LargePartitions = Requirement(
+ name="RQ.ClickHouse.ExportPartition.LargePartitions",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support exporting large partitions by:\n"
+ "* Handling partitions with large numbers of parts (e.g., hundreds or thousands of parts)\n"
+ "* Processing partitions with large numbers of rows (e.g., billions of rows)\n"
+ "* Processing large data volumes efficiently during export\n"
+ "* Maintaining data integrity when exporting large partitions\n"
+ "* Completing export operations successfully regardless of partition size\n"
+ "* Allowing export operations to continue over extended periods of time for very large partitions\n"
+ "\n"
+ "Production systems often have partitions containing very large amounts of data, and the export must handle these efficiently without timeouts or memory issues.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="12.3",
+)
+
+RQ_ClickHouse_ExportPartition_Corrupted = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Corrupted",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL output an error and prevent export operations from proceeding when trying to export a partition that contains corrupted parts in the source table.\n"
+ "\n"
+ "The system SHALL detect corruption in partitions containing compact parts, wide parts, or mixed part types.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="12.4",
+)
+
+RQ_ClickHouse_ExportPartition_RetryMechanism = Requirement(
+ name="RQ.ClickHouse.ExportPartition.RetryMechanism",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL automatically retry failed part exports within a partition up to a configurable maximum retry count. If all retry attempts are exhausted for a part, the entire partition export operation SHALL be marked as failed.\n"
+ "\n"
+ "Unlike single-part exports, partition exports involve multiple parts and may take significant time. Retry mechanisms ensure that temporary failures don't require restarting the entire export operation.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.1",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_MaxRetries = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.MaxRetries",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `export_merge_tree_partition_max_retries` setting that controls the maximum number of retries for exporting a merge tree part in an export partition task. The default value SHALL be `3`.\n"
+ "\n"
+ "This setting allows users to control how many times the system will retry exporting a part before marking it as failed.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1,\n"
+ " export_merge_tree_partition_max_retries = 5\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.2",
+)
+
+RQ_ClickHouse_ExportPartition_ResumeAfterFailure = Requirement(
+ name="RQ.ClickHouse.ExportPartition.ResumeAfterFailure",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL allow export operations to resume after node failures or restarts. The system SHALL track which parts have been successfully exported and SHALL not re-export parts that were already successfully exported.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.3",
+)
+
+RQ_ClickHouse_ExportPartition_PartialProgress = Requirement(
+ name="RQ.ClickHouse.ExportPartition.PartialProgress",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL allow export operations to make partial progress, with successfully exported parts remaining in the destination even if other parts fail. Users SHALL be able to see which parts have been successfully exported and which parts have failed.\n"
+ "\n"
+ "For example, users can query the export status to see partial progress:\n"
+ "\n"
+ "```sql\n"
+ "SELECT source_table, destination_table, partition_id, status,\n"
+ " parts_total, parts_processed, parts_failed\n"
+ "FROM system.replicated_partition_exports\n"
+ "WHERE partition_id = '2020'\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.4",
+)
+
+RQ_ClickHouse_ExportPartition_Cleanup = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Cleanup",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL automatically clean up failed or completed export operations after a configurable TTL period.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.5",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_ManifestTTL = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.ManifestTTL",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `export_merge_tree_partition_manifest_ttl` setting that determines how long the export manifest will be retained. This setting prevents the same partition from being exported twice to the same destination within the TTL period. The default value SHALL be `180` seconds.\n"
+ "\n"
+ "This setting only affects completed export operations and does not delete in-progress tasks. It allows users to control how long export history is maintained to prevent duplicate exports.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1,\n"
+ " export_merge_tree_partition_manifest_ttl = 360\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="13.6",
+)
+
+RQ_ClickHouse_ExportPartition_NetworkResilience_PacketIssues = Requirement(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle network packet issues during export operations by:\n"
+ "* Tolerating packet delay without data corruption or loss\n"
+ "* Handling packet loss and retransmitting data as needed\n"
+ "* Detecting and handling packet corruption to ensure data integrity\n"
+ "* Managing packet duplication without data duplication in destination\n"
+ "* Handling packet reordering to maintain correct data sequence\n"
+ "* Operating correctly under packet rate limiting constraints\n"
+ "* Completing exports successfully despite network impairments\n"
+ "\n"
+ "Network issues are common in distributed systems, and export operations must be resilient to ensure data integrity.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="14.1",
+)
+
+RQ_ClickHouse_ExportPartition_NetworkResilience_DestinationInterruption = Requirement(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle destination storage interruptions during export operations by:\n"
+ "* Detecting when destination storage becomes unavailable during export\n"
+ "* Retrying failed part exports when destination storage becomes available again\n"
+ "* Logging failed exports in the `system.events` table with appropriate counters\n"
+ "* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability\n"
+ "* Allowing exports to complete successfully once destination storage becomes available again\n"
+ "* Resuming export operations from the last successfully exported part\n"
+ "\n"
+ "Destination storage systems may experience temporary outages, and the export should automatically recover when service is restored.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="14.2",
+)
+
+RQ_ClickHouse_ExportPartition_NetworkResilience_NodeInterruption = Requirement(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:\n"
+ "* Allowing export operations to resume after node restart without data loss or duplication\n"
+ "* Allowing other replicas to continue or complete export operations if a node fails\n"
+ "* Not leaving partial or corrupted data in destination storage when node restarts occur\n"
+ "* With safe shutdown, ensuring exports complete successfully before node shutdown when possible\n"
+ "* With unsafe shutdown, allowing export operations to resume from the last checkpoint after node restart\n"
+ "* Maintaining data integrity in destination storage regardless of node interruption type\n"
+ "* Ensuring that parts already exported are not re-exported after node restart\n"
+ "\n"
+ "Node failures are common in distributed systems, and export operations must be able to recover and continue without data loss or duplication.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="14.3",
+)
+
+RQ_ClickHouse_ExportPartition_Restrictions_SameTable = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.SameTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL prevent exporting partitions to the same table as the source by:\n"
+ "* Validating that source and destination table identifiers are different\n"
+ '* Throwing a `BAD_ARGUMENTS` exception with message "Exporting to the same table is not allowed" when source and destination are identical\n'
+ "* Performing this validation before any export processing begins\n"
+ "\n"
+ "Exporting to the same table would be redundant and could cause data duplication or conflicts.\n"
+ "\n"
+ "For example, the following command SHALL output an error:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE my_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE my_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="15.1.1",
+)
+
+RQ_ClickHouse_ExportPartition_Restrictions_DestinationSupport = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate destination table compatibility by:\n"
+ "\n"
+ "* Checking that the destination storage supports importing MergeTree parts\n"
+ "* Verifying that the destination uses Hive partitioning strategy (`partition_strategy = 'hive'`)\n"
+ '* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ "The destination must support the format and partitioning strategy required for exported data.\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="15.2.1",
+)
+
+RQ_ClickHouse_ExportPartition_Restrictions_LocalTable = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.LocalTable",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL prevent exporting partitions to local MergeTree tables by:\n"
+ "* Rejecting export operations where the destination table uses a MergeTree engine\n"
+ '* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ "Export partition is designed to move data to object storage, not to local MergeTree tables.\n"
+ "\n"
+ "For example, if `local_table` is a MergeTree table, the following command SHALL output an error:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE local_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="15.3.1",
+)
+
+RQ_ClickHouse_ExportPartition_Restrictions_PartitionKey = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:\n"
+ "* Checking that the partition key expression matches between source and destination tables\n"
+ '* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ\n'
+ "* Performing this validation during the initial export setup phase\n"
+ "\n"
+ "Matching partition keys ensure that exported data is organized correctly in the destination storage.\n"
+ "\n"
+ "For example, if `source_table` is partitioned by `toYYYYMM(date)` and `destination_table` is partitioned by `toYYYYMMDD(date)`, the following command SHALL output an error:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="15.4.1",
+)
+
+RQ_ClickHouse_ExportPartition_Restrictions_SourcePartition = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL validate source partition availability by:\n"
+ "* Checking that the specified partition ID exists in the source table\n"
+ "* Verifying that the partition contains at least one active part (not detached or missing)\n"
+ "* Throwing an exception with an appropriate error message when the partition is not found or is empty\n"
+ "* Performing this validation before any export processing begins\n"
+ "\n"
+ "The system must verify that the partition exists and contains data before attempting to export it.\n"
+ "\n"
+ "For example, if partition ID '2025' does not exist in `source_table`, the following command SHALL output an error:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2025' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=3,
+ num="15.5.1",
+)
+
+RQ_ClickHouse_ExportPartition_Concurrency = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Concurrency",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support concurrent export operations by:\n"
+ "* Allowing multiple partition exports to run simultaneously without interference\n"
+ "* Supporting concurrent exports of different partitions to different destinations\n"
+ "* Preventing concurrent exports of the same partition to the same destination\n"
+ "* Allowing different replicas to export different parts of the same partition concurrently\n"
+ "* Maintaining separate progress tracking for each concurrent operation\n"
+ "\n"
+ "Multiple users may want to export different partitions simultaneously, and the system must coordinate these operations to prevent conflicts while maximizing parallelism.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="16.1",
+)
+
+RQ_ClickHouse_ExportPartition_Idempotency = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Idempotency",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL handle duplicate export operations by:\n"
+ "* Preventing duplicate data from being exported when the same partition is exported multiple times to the same destination\n"
+ "* Detecting when a partition export is already in progress or completed\n"
+ "* Detecting when an export operation attempts to export a partition that already exists in the destination\n"
+ "* Logging duplicate export attempts in the `system.events` table with appropriate counters\n"
+ "* Ensuring that destination data matches source data without duplication when the same partition is exported multiple times\n"
+ "* Allowing users to force re-export of a partition if needed (e.g., after TTL expiration or manual cleanup)\n"
+ "\n"
+ "Users may accidentally trigger the same export multiple times, and the system should prevent duplicate data while allowing legitimate re-exports when needed.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="17.1",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_ForceExport = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.ForceExport",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `export_merge_tree_partition_force_export` setting that allows users to ignore existing partition export entries and force a new export operation. The default value SHALL be `false` (turned off).\n"
+ "\n"
+ "When set to `true`, this setting allows users to overwrite existing export entries and force re-export of a partition, even if a previous export operation exists for the same partition and destination.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1,\n"
+ " export_merge_tree_partition_force_export = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="17.2",
+)
+
+RQ_ClickHouse_ExportPartition_Logging = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Logging",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide detailed logging for export operations by:\n"
+ "* Logging all export operations (both successful and failed) with timestamps and details\n"
+ "* Recording the specific partition ID in the `system.part_log` table for all operations\n"
+ "* Logging export events in the `system.events` table, including:\n"
+ " * `PartsExports` - Number of successful part exports (within partitions)\n"
+ " * `PartsExportFailures` - Number of failed part exports\n"
+ " * `PartsExportDuplicated` - Number of part exports that failed because target already exists\n"
+ "* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PARTITION`\n"
+ "* Providing sufficient detail for monitoring and troubleshooting export operations\n"
+ "* Logging per-part export status within partition exports\n"
+ "\n"
+ "Detailed logging helps users monitor export progress, troubleshoot issues, and audit export operations.\n"
+ "\n"
+ "For example, users can query export logs:\n"
+ "\n"
+ "```sql\n"
+ "SELECT event_time, event_type, table, partition, rows, bytes_read, bytes_written\n"
+ "FROM system.part_log\n"
+ "WHERE event_type = 'EXPORT_PARTITION'\n"
+ "ORDER BY event_time DESC\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="18.1",
+)
+
+RQ_ClickHouse_ExportPartition_SystemTables_Exports = Requirement(
+ name="RQ.ClickHouse.ExportPartition.SystemTables.Exports",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide a `system.replicated_partition_exports` table that allows users to monitor active partition export operations with at least the following columns:\n"
+ "* `source_table` - source table identifier\n"
+ "* `destination_table` - destination table identifier\n"
+ "* `partition_id` - the partition ID being exported\n"
+ "* `status` - current status of the export operation (e.g., PENDING, IN_PROGRESS, COMPLETED, FAILED)\n"
+ "* `parts_total` - total number of parts in the partition\n"
+ "* `parts_processed` - number of parts successfully exported\n"
+ "* `parts_failed` - number of parts that failed to export\n"
+ "* `create_time` - when the export operation was created\n"
+ "* `update_time` - last update time of the export operation\n"
+ "\n"
+ "The table SHALL track export operations before they complete and SHALL show completed or failed exports until they are cleaned up (based on TTL).\n"
+ "\n"
+ "Users need visibility into export operations to monitor progress, identify issues, and understand export status across the cluster.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "SELECT source_table, destination_table, partition_id, status, \n"
+ " parts_total, parts_processed, parts_failed, create_time, update_time\n"
+ "FROM system.replicated_partition_exports\n"
+ "WHERE status = 'IN_PROGRESS'\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="19.1",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_AllowExperimental = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.AllowExperimental",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `allow_experimental_export_merge_tree_part` setting that SHALL gate the experimental export partition functionality, which SHALL be set to `1` to enable `ALTER TABLE ... EXPORT PARTITION ID ...` commands. The default value SHALL be `0` (turned off).\n"
+ "\n"
+ "This setting allows administrators to control access to experimental functionality and ensures users are aware they are using a feature that may change.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="20.1",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_AllowExperimental_Disabled = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL prevent export partition operations when `allow_experimental_export_merge_tree_part` is set to `0` (turned off). When the setting is `0`, attempting to execute `ALTER TABLE ... EXPORT PARTITION ID ...` commands SHALL result in an error indicating that the experimental feature is not enabled.\n"
+ "\n"
+ "For example, the following command SHALL output an error when the setting is `0`:\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="20.2",
+)
+
+RQ_ClickHouse_ExportPartition_Settings_OverwriteFile = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Settings.OverwriteFile",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `export_merge_tree_part_overwrite_file_if_exists` setting that controls whether to overwrite files if they already exist when exporting a partition. The default value SHALL be `0` (turned off), meaning exports will fail if files already exist in the destination.\n"
+ "\n"
+ "This setting allows users to control whether to overwrite existing data in the destination, providing safety by default while allowing overwrites when needed.\n"
+ "\n"
+ "For example,\n"
+ "\n"
+ "```sql\n"
+ "ALTER TABLE source_table \n"
+ "EXPORT PARTITION ID '2020' \n"
+ "TO TABLE destination_table\n"
+ "SETTINGS allow_experimental_export_merge_tree_part = 1,\n"
+ " export_merge_tree_part_overwrite_file_if_exists = 1\n"
+ "```\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="21.1",
+)
+
+RQ_ClickHouse_ExportPartition_ParallelFormatting = Requirement(
+ name="RQ.ClickHouse.ExportPartition.ParallelFormatting",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support parallel formatting for export operations by:\n"
+ "* Automatically enabling parallel formatting for large export operations to improve performance\n"
+ "* Using the `output_format_parallel_formatting` setting to control parallel formatting behavior\n"
+ "* Optimizing data processing based on export size and system resources\n"
+ "* Providing consistent formatting performance across different export scenarios\n"
+ "* Allowing parallel processing of multiple parts within a partition when possible\n"
+ "\n"
+ "Parallel formatting improves export performance, especially for large partitions with many parts.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="22.1",
+)
+
+RQ_ClickHouse_ExportPartition_ServerSettings_MaxBandwidth = Requirement(
+ name="RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.\n"
+ "\n"
+ "Administrators need to control export bandwidth to avoid impacting other operations on the server.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="23.1",
+)
+
+RQ_ClickHouse_ExportPartition_ServerSettings_BackgroundMovePoolSize = Requirement(
+ name="RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.\n"
+ "\n"
+ "This setting allows administrators to balance export performance with other system operations.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="23.2",
+)
+
+RQ_ClickHouse_ExportPartition_Metrics_Export = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Metrics.Export",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL provide the `Export` current metric in `system.metrics` table that tracks the number of currently executing partition exports.\n"
+ "\n"
+ "This metric helps monitor system load from export operations.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="23.3",
+)
+
+RQ_ClickHouse_ExportPartition_Security_RBAC = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Security.RBAC",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL enforce role-based access control (RBAC) for export operations. Users must have the following privileges to perform export operations:\n"
+ "* **Source Table**: `SELECT` privilege on the source table to read data parts\n"
+ "* **Destination Table**: `INSERT` privilege on the destination table to write exported data\n"
+ "* **Database Access**: `SHOW` privilege on both source and destination databases\n"
+ "* **System Tables**: `SELECT` privilege on `system.tables` and `system.replicated_partition_exports` to validate table existence and monitor exports\n"
+ "\n"
+ "Export operations move potentially sensitive data, and proper access controls ensure only authorized users can export partitions.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="24.1",
+)
+
+RQ_ClickHouse_ExportPartition_Security_DataEncryption = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Security.DataEncryption",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL encrypt all data in transit to destination storage using TLS/SSL during export operations.\n"
+ "\n"
+ "Data encryption protects sensitive information from being intercepted or accessed during transmission to destination storage.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="24.2",
+)
+
+RQ_ClickHouse_ExportPartition_Security_Network = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Security.Network",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL use secure connections to destination storage during export operations. For S3-compatible storage, connections must use HTTPS. For other storage types, secure protocols appropriate to the storage system must be used.\n"
+ "\n"
+ "Secure network connections prevent unauthorized access and ensure data integrity during export operations.\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="24.3",
+)
+
+RQ_ClickHouse_ExportPartition_Security_CredentialManagement = Requirement(
+ name="RQ.ClickHouse.ExportPartition.Security.CredentialManagement",
+ version="1.0",
+ priority=None,
+ group=None,
+ type=None,
+ uid=None,
+ description=(
+ "[ClickHouse] SHALL use secure credential storage for export operations and SHALL avoid exposing credentials in logs or error messages.\n"
+ "\n"
+ "Proper credential management prevents unauthorized access to destination storage systems and protects sensitive authentication information.\n"
+ "\n"
+ "\n"
+ "[ClickHouse]: https://clickhouse.com\n"
+ "\n"
+ ),
+ link=None,
+ level=2,
+ num="24.4",
+)
+
+SRS_016_ClickHouse_Export_Partition_to_S3 = Specification(
+ name="SRS-016 ClickHouse Export Partition to S3",
+ description=None,
+ author=None,
+ date=None,
+ status=None,
+ approved_by=None,
+ approved_date=None,
+ approved_version=None,
+ version=None,
+ group=None,
+ type=None,
+ link=None,
+ uid=None,
+ parent=None,
+ children=None,
+ headings=(
+ Heading(name="Introduction", level=1, num="1"),
+ Heading(name="Exporting Partitions to S3", level=1, num="2"),
+ Heading(name="RQ.ClickHouse.ExportPartition.S3", level=2, num="2.1"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.EmptyPartition", level=2, num="2.2"
+ ),
+ Heading(name="SQL command support", level=1, num="3"),
+ Heading(name="RQ.ClickHouse.ExportPartition.SQLCommand", level=2, num="3.1"),
+ Heading(name="RQ.ClickHouse.ExportPartition.IntoOutfile", level=2, num="3.2"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Format", level=2, num="3.3"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.SettingsClause", level=2, num="3.4"
+ ),
+ Heading(name="Supported source table engines", level=1, num="4"),
+ Heading(name="RQ.ClickHouse.ExportPartition.SourceEngines", level=2, num="4.1"),
+ Heading(name="Cluster and node support", level=1, num="5"),
+ Heading(name="RQ.ClickHouse.ExportPartition.ClustersNodes", level=2, num="5.1"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Shards", level=2, num="5.2"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Versions", level=2, num="5.3"),
+ Heading(name="Supported source part storage types", level=1, num="6"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.SourcePartStorage", level=2, num="6.1"
+ ),
+ Heading(name="Storage policies and volumes", level=1, num="7"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.StoragePolicies", level=2, num="7.1"
+ ),
+ Heading(name="Supported destination table engines", level=1, num="8"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.DestinationEngines", level=2, num="8.1"
+ ),
+ Heading(name="Temporary tables", level=1, num="9"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.TemporaryTable", level=2, num="9.1"
+ ),
+ Heading(name="Schema compatibility", level=1, num="10"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.SchemaCompatibility",
+ level=2,
+ num="10.1",
+ ),
+ Heading(name="Partition key types support", level=1, num="11"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.PartitionKeyTypes", level=2, num="11.1"
+ ),
+ Heading(name="Partition content support", level=1, num="12"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.PartitionContent", level=2, num="12.1"
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.SchemaChangeIsolation",
+ level=2,
+ num="12.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.LargePartitions", level=2, num="12.3"
+ ),
+ Heading(name="RQ.ClickHouse.ExportPartition.Corrupted", level=2, num="12.4"),
+ Heading(name="Export operation failure handling", level=1, num="13"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.RetryMechanism", level=2, num="13.1"
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.MaxRetries",
+ level=2,
+ num="13.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.ResumeAfterFailure", level=2, num="13.3"
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.PartialProgress", level=2, num="13.4"
+ ),
+ Heading(name="RQ.ClickHouse.ExportPartition.Cleanup", level=2, num="13.5"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.ManifestTTL",
+ level=2,
+ num="13.6",
+ ),
+ Heading(name="Network resilience", level=1, num="14"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues",
+ level=2,
+ num="14.1",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption",
+ level=2,
+ num="14.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption",
+ level=2,
+ num="14.3",
+ ),
+ Heading(name="Export operation restrictions", level=1, num="15"),
+ Heading(name="Preventing same table exports", level=2, num="15.1"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.SameTable",
+ level=3,
+ num="15.1.1",
+ ),
+ Heading(name="Destination table compatibility", level=2, num="15.2"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport",
+ level=3,
+ num="15.2.1",
+ ),
+ Heading(name="Local table restriction", level=2, num="15.3"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.LocalTable",
+ level=3,
+ num="15.3.1",
+ ),
+ Heading(name="Partition key compatibility", level=2, num="15.4"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey",
+ level=3,
+ num="15.4.1",
+ ),
+ Heading(name="Source partition availability", level=2, num="15.5"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition",
+ level=3,
+ num="15.5.1",
+ ),
+ Heading(name="Export operation concurrency", level=1, num="16"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Concurrency", level=2, num="16.1"),
+ Heading(name="Export operation idempotency", level=1, num="17"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Idempotency", level=2, num="17.1"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.ForceExport",
+ level=2,
+ num="17.2",
+ ),
+ Heading(name="Export operation logging", level=1, num="18"),
+ Heading(name="RQ.ClickHouse.ExportPartition.Logging", level=2, num="18.1"),
+ Heading(name="Monitoring export operations", level=1, num="19"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.SystemTables.Exports",
+ level=2,
+ num="19.1",
+ ),
+ Heading(name="Enabling export functionality", level=1, num="20"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.AllowExperimental",
+ level=2,
+ num="20.1",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled",
+ level=2,
+ num="20.2",
+ ),
+ Heading(name="Handling file conflicts during export", level=1, num="21"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Settings.OverwriteFile",
+ level=2,
+ num="21.1",
+ ),
+ Heading(name="Export operation configuration", level=1, num="22"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.ParallelFormatting", level=2, num="22.1"
+ ),
+ Heading(name="Controlling export performance", level=1, num="23"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth",
+ level=2,
+ num="23.1",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize",
+ level=2,
+ num="23.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Metrics.Export", level=2, num="23.3"
+ ),
+ Heading(name="Export operation security", level=1, num="24"),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Security.RBAC", level=2, num="24.1"
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Security.DataEncryption",
+ level=2,
+ num="24.2",
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Security.Network", level=2, num="24.3"
+ ),
+ Heading(
+ name="RQ.ClickHouse.ExportPartition.Security.CredentialManagement",
+ level=2,
+ num="24.4",
+ ),
+ ),
+ requirements=(
+ RQ_ClickHouse_ExportPartition_S3,
+ RQ_ClickHouse_ExportPartition_EmptyPartition,
+ RQ_ClickHouse_ExportPartition_SQLCommand,
+ RQ_ClickHouse_ExportPartition_IntoOutfile,
+ RQ_ClickHouse_ExportPartition_Format,
+ RQ_ClickHouse_ExportPartition_SettingsClause,
+ RQ_ClickHouse_ExportPartition_SourceEngines,
+ RQ_ClickHouse_ExportPartition_ClustersNodes,
+ RQ_ClickHouse_ExportPartition_Shards,
+ RQ_ClickHouse_ExportPartition_Versions,
+ RQ_ClickHouse_ExportPartition_SourcePartStorage,
+ RQ_ClickHouse_ExportPartition_StoragePolicies,
+ RQ_ClickHouse_ExportPartition_DestinationEngines,
+ RQ_ClickHouse_ExportPartition_TemporaryTable,
+ RQ_ClickHouse_ExportPartition_SchemaCompatibility,
+ RQ_ClickHouse_ExportPartition_PartitionKeyTypes,
+ RQ_ClickHouse_ExportPartition_PartitionContent,
+ RQ_ClickHouse_ExportPartition_SchemaChangeIsolation,
+ RQ_ClickHouse_ExportPartition_LargePartitions,
+ RQ_ClickHouse_ExportPartition_Corrupted,
+ RQ_ClickHouse_ExportPartition_RetryMechanism,
+ RQ_ClickHouse_ExportPartition_Settings_MaxRetries,
+ RQ_ClickHouse_ExportPartition_ResumeAfterFailure,
+ RQ_ClickHouse_ExportPartition_PartialProgress,
+ RQ_ClickHouse_ExportPartition_Cleanup,
+ RQ_ClickHouse_ExportPartition_Settings_ManifestTTL,
+ RQ_ClickHouse_ExportPartition_NetworkResilience_PacketIssues,
+ RQ_ClickHouse_ExportPartition_NetworkResilience_DestinationInterruption,
+ RQ_ClickHouse_ExportPartition_NetworkResilience_NodeInterruption,
+ RQ_ClickHouse_ExportPartition_Restrictions_SameTable,
+ RQ_ClickHouse_ExportPartition_Restrictions_DestinationSupport,
+ RQ_ClickHouse_ExportPartition_Restrictions_LocalTable,
+ RQ_ClickHouse_ExportPartition_Restrictions_PartitionKey,
+ RQ_ClickHouse_ExportPartition_Restrictions_SourcePartition,
+ RQ_ClickHouse_ExportPartition_Concurrency,
+ RQ_ClickHouse_ExportPartition_Idempotency,
+ RQ_ClickHouse_ExportPartition_Settings_ForceExport,
+ RQ_ClickHouse_ExportPartition_Logging,
+ RQ_ClickHouse_ExportPartition_SystemTables_Exports,
+ RQ_ClickHouse_ExportPartition_Settings_AllowExperimental,
+ RQ_ClickHouse_ExportPartition_Settings_AllowExperimental_Disabled,
+ RQ_ClickHouse_ExportPartition_Settings_OverwriteFile,
+ RQ_ClickHouse_ExportPartition_ParallelFormatting,
+ RQ_ClickHouse_ExportPartition_ServerSettings_MaxBandwidth,
+ RQ_ClickHouse_ExportPartition_ServerSettings_BackgroundMovePoolSize,
+ RQ_ClickHouse_ExportPartition_Metrics_Export,
+ RQ_ClickHouse_ExportPartition_Security_RBAC,
+ RQ_ClickHouse_ExportPartition_Security_DataEncryption,
+ RQ_ClickHouse_ExportPartition_Security_Network,
+ RQ_ClickHouse_ExportPartition_Security_CredentialManagement,
+ ),
+ content=r"""
+# SRS-016 ClickHouse Export Partition to S3
+# Software Requirements Specification
+
+## Table of Contents
+
+* 1 [Introduction](#introduction)
+* 2 [Exporting Partitions to S3](#exporting-partitions-to-s3)
+ * 2.1 [RQ.ClickHouse.ExportPartition.S3](#rqclickhouseexportpartitions3)
+ * 2.2 [RQ.ClickHouse.ExportPartition.EmptyPartition](#rqclickhouseexportpartitionemptypartition)
+* 3 [SQL command support](#sql-command-support)
+ * 3.1 [RQ.ClickHouse.ExportPartition.SQLCommand](#rqclickhouseexportpartitionsqlcommand)
+ * 3.2 [RQ.ClickHouse.ExportPartition.IntoOutfile](#rqclickhouseexportpartitionintooutfile)
+ * 3.3 [RQ.ClickHouse.ExportPartition.Format](#rqclickhouseexportpartitionformat)
+ * 3.4 [RQ.ClickHouse.ExportPartition.SettingsClause](#rqclickhouseexportpartitionsettingsclause)
+* 4 [Supported source table engines](#supported-source-table-engines)
+ * 4.1 [RQ.ClickHouse.ExportPartition.SourceEngines](#rqclickhouseexportpartitionsourceengines)
+* 5 [Cluster and node support](#cluster-and-node-support)
+ * 5.1 [RQ.ClickHouse.ExportPartition.ClustersNodes](#rqclickhouseexportpartitionclustersnodes)
+ * 5.2 [RQ.ClickHouse.ExportPartition.Shards](#rqclickhouseexportpartitionshards)
+ * 5.3 [RQ.ClickHouse.ExportPartition.Versions](#rqclickhouseexportpartitionversions)
+* 6 [Supported source part storage types](#supported-source-part-storage-types)
+ * 6.1 [RQ.ClickHouse.ExportPartition.SourcePartStorage](#rqclickhouseexportpartitionsourcepartstorage)
+* 7 [Storage policies and volumes](#storage-policies-and-volumes)
+ * 7.1 [RQ.ClickHouse.ExportPartition.StoragePolicies](#rqclickhouseexportpartitionstoragepolicies)
+* 8 [Supported destination table engines](#supported-destination-table-engines)
+ * 8.1 [RQ.ClickHouse.ExportPartition.DestinationEngines](#rqclickhouseexportpartitiondestinationengines)
+* 9 [Temporary tables](#temporary-tables)
+ * 9.1 [RQ.ClickHouse.ExportPartition.TemporaryTable](#rqclickhouseexportpartitiontemporarytable)
+* 10 [Schema compatibility](#schema-compatibility)
+ * 10.1 [RQ.ClickHouse.ExportPartition.SchemaCompatibility](#rqclickhouseexportpartitionschemacompatibility)
+* 11 [Partition key types support](#partition-key-types-support)
+ * 11.1 [RQ.ClickHouse.ExportPartition.PartitionKeyTypes](#rqclickhouseexportpartitionpartitionkeytypes)
+* 12 [Partition content support](#partition-content-support)
+ * 12.1 [RQ.ClickHouse.ExportPartition.PartitionContent](#rqclickhouseexportpartitionpartitioncontent)
+ * 12.2 [RQ.ClickHouse.ExportPartition.SchemaChangeIsolation](#rqclickhouseexportpartitionschemachangeisolation)
+ * 12.3 [RQ.ClickHouse.ExportPartition.LargePartitions](#rqclickhouseexportpartitionlargepartitions)
+ * 12.4 [RQ.ClickHouse.ExportPartition.Corrupted](#rqclickhouseexportpartitioncorrupted)
+* 13 [Export operation failure handling](#export-operation-failure-handling)
+ * 13.1 [RQ.ClickHouse.ExportPartition.RetryMechanism](#rqclickhouseexportpartitionretrymechanism)
+ * 13.2 [RQ.ClickHouse.ExportPartition.Settings.MaxRetries](#rqclickhouseexportpartitionsettingsmaxretries)
+ * 13.3 [RQ.ClickHouse.ExportPartition.ResumeAfterFailure](#rqclickhouseexportpartitionresumeafterfailure)
+ * 13.4 [RQ.ClickHouse.ExportPartition.PartialProgress](#rqclickhouseexportpartitionpartialprogress)
+ * 13.5 [RQ.ClickHouse.ExportPartition.Cleanup](#rqclickhouseexportpartitioncleanup)
+ * 13.6 [RQ.ClickHouse.ExportPartition.Settings.ManifestTTL](#rqclickhouseexportpartitionsettingsmanifestttl)
+* 14 [Network resilience](#network-resilience)
+ * 14.1 [RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues](#rqclickhouseexportpartitionnetworkresiliencepacketissues)
+ * 14.2 [RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption](#rqclickhouseexportpartitionnetworkresiliencedestinationinterruption)
+ * 14.3 [RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption](#rqclickhouseexportpartitionnetworkresiliencenodeinterruption)
+* 15 [Export operation restrictions](#export-operation-restrictions)
+ * 15.1 [Preventing same table exports](#preventing-same-table-exports)
+ * 15.1.1 [RQ.ClickHouse.ExportPartition.Restrictions.SameTable](#rqclickhouseexportpartitionrestrictionssametable)
+ * 15.2 [Destination table compatibility](#destination-table-compatibility)
+ * 15.2.1 [RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport](#rqclickhouseexportpartitionrestrictionsdestinationsupport)
+ * 15.3 [Local table restriction](#local-table-restriction)
+ * 15.3.1 [RQ.ClickHouse.ExportPartition.Restrictions.LocalTable](#rqclickhouseexportpartitionrestrictionslocaltable)
+ * 15.4 [Partition key compatibility](#partition-key-compatibility)
+ * 15.4.1 [RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey](#rqclickhouseexportpartitionrestrictionspartitionkey)
+ * 15.5 [Source partition availability](#source-partition-availability)
+ * 15.5.1 [RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition](#rqclickhouseexportpartitionrestrictionssourcepartition)
+* 16 [Export operation concurrency](#export-operation-concurrency)
+ * 16.1 [RQ.ClickHouse.ExportPartition.Concurrency](#rqclickhouseexportpartitionconcurrency)
+* 17 [Export operation idempotency](#export-operation-idempotency)
+ * 17.1 [RQ.ClickHouse.ExportPartition.Idempotency](#rqclickhouseexportpartitionidempotency)
+ * 17.2 [RQ.ClickHouse.ExportPartition.Settings.ForceExport](#rqclickhouseexportpartitionsettingsforceexport)
+* 18 [Export operation logging](#export-operation-logging)
+ * 18.1 [RQ.ClickHouse.ExportPartition.Logging](#rqclickhouseexportpartitionlogging)
+* 19 [Monitoring export operations](#monitoring-export-operations)
+ * 19.1 [RQ.ClickHouse.ExportPartition.SystemTables.Exports](#rqclickhouseexportpartitionsystemtablesexports)
+* 20 [Enabling export functionality](#enabling-export-functionality)
+ * 20.1 [RQ.ClickHouse.ExportPartition.Settings.AllowExperimental](#rqclickhouseexportpartitionsettingsallowexperimental)
+ * 20.2 [RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled](#rqclickhouseexportpartitionsettingsallowexperimentaldisabled)
+* 21 [Handling file conflicts during export](#handling-file-conflicts-during-export)
+ * 21.1 [RQ.ClickHouse.ExportPartition.Settings.OverwriteFile](#rqclickhouseexportpartitionsettingsoverwritefile)
+* 22 [Export operation configuration](#export-operation-configuration)
+ * 22.1 [RQ.ClickHouse.ExportPartition.ParallelFormatting](#rqclickhouseexportpartitionparallelformatting)
+* 23 [Controlling export performance](#controlling-export-performance)
+ * 23.1 [RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth](#rqclickhouseexportpartitionserversettingsmaxbandwidth)
+ * 23.2 [RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize](#rqclickhouseexportpartitionserversettingsbackgroundmovepoolsize)
+ * 23.3 [RQ.ClickHouse.ExportPartition.Metrics.Export](#rqclickhouseexportpartitionmetricsexport)
+* 24 [Export operation security](#export-operation-security)
+ * 24.1 [RQ.ClickHouse.ExportPartition.Security.RBAC](#rqclickhouseexportpartitionsecurityrbac)
+ * 24.2 [RQ.ClickHouse.ExportPartition.Security.DataEncryption](#rqclickhouseexportpartitionsecuritydataencryption)
+ * 24.3 [RQ.ClickHouse.ExportPartition.Security.Network](#rqclickhouseexportpartitionsecuritynetwork)
+ * 24.4 [RQ.ClickHouse.ExportPartition.Security.CredentialManagement](#rqclickhouseexportpartitionsecuritycredentialmanagement)
+
+## Introduction
+
+This specification defines requirements for exporting partitions (all parts within a partition) from ReplicatedMergeTree tables to S3-compatible object storage. This feature enables users to export entire partitions containing multiple data parts across cluster nodes.
+
+## Exporting Partitions to S3
+
+### RQ.ClickHouse.ExportPartition.S3
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions (all parts within a partition) from ReplicatedMergeTree engine tables to S3 object storage. The export operation SHALL export all parts that belong to the specified partition ID, ensuring complete partition data is transferred to the destination.
+
+### RQ.ClickHouse.ExportPartition.EmptyPartition
+version: 1.0
+
+[ClickHouse] SHALL support exporting from empty partitions by:
+* Completing export operations successfully when the specified partition contains no parts
+* Resulting in an empty destination partition when exporting from an empty source partition
+* Not creating any files in destination storage when there are no parts to export in the partition
+* Handling empty partitions gracefully without errors
+
+## SQL command support
+
+### RQ.ClickHouse.ExportPartition.SQLCommand
+version: 1.0
+
+[ClickHouse] SHALL support the following SQL command syntax for exporting partitions from ReplicatedMergeTree tables to object storage tables:
+
+```sql
+ALTER TABLE [database.]source_table_name
+EXPORT PARTITION ID 'partition_id'
+TO TABLE [database.]destination_table_name
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+**Parameters:**
+- `source_table_name`: Name of the source ReplicatedMergeTree table
+- `partition_id`: The partition ID to export (string literal), which identifies all parts belonging to that partition
+- `destination_table_name`: Name of the destination object storage table
+- `allow_experimental_export_merge_tree_part`: Setting that must be set to `1` to enable this experimental feature
+
+This command allows users to export entire partitions in a single operation, which is more efficient than exporting individual parts and ensures all data for a partition is exported together.
+
+### RQ.ClickHouse.ExportPartition.IntoOutfile
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `INTO OUTFILE` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+INTO OUTFILE '/path/to/file'
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### RQ.ClickHouse.ExportPartition.Format
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `FORMAT` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+FORMAT JSON
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### RQ.ClickHouse.ExportPartition.SettingsClause
+version: 1.0
+
+[ClickHouse] SHALL support the usage of the `SETTINGS` clause with `EXPORT PARTITION` and SHALL not output any errors.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_max_retries = 5
+```
+
+## Supported source table engines
+
+### RQ.ClickHouse.ExportPartition.SourceEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from the following source table engines:
+* `ReplicatedMergeTree` - Replicated MergeTree engine (primary use case)
+* `ReplicatedSummingMergeTree` - Replicated MergeTree with automatic summation
+* `ReplicatedAggregatingMergeTree` - Replicated MergeTree with pre-aggregated data
+* `ReplicatedCollapsingMergeTree` - Replicated MergeTree with row versioning
+* `ReplicatedVersionedCollapsingMergeTree` - Replicated CollapsingMergeTree with version tracking
+* `ReplicatedGraphiteMergeTree` - Replicated MergeTree optimized for Graphite data
+* All other ReplicatedMergeTree family engines
+
+Export partition functionality manages export operations across multiple replicas in a cluster, ensuring that parts are exported correctly and avoiding conflicts.
+
+## Cluster and node support
+
+### RQ.ClickHouse.ExportPartition.ClustersNodes
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from multiple nodes in a ReplicatedMergeTree cluster to the same destination storage, ensuring that:
+* Each replica in the cluster can independently export parts from the partition that it owns locally
+* All parts within a partition are exported exactly once, even when distributed across multiple replicas
+* Exported data from different replicas is correctly aggregated in the destination storage
+* All nodes in the cluster can read the same exported partition data from the destination
+* Export operations continue to make progress even if some replicas are temporarily unavailable
+
+In a replicated cluster, different parts of the same partition may exist on different replicas. The system must coordinate exports across all replicas to ensure complete partition export without duplication.
+
+### RQ.ClickHouse.ExportPartition.Shards
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from source tables that are on different shards than the destination table.
+
+### RQ.ClickHouse.ExportPartition.Versions
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from source tables that are stored on servers with different ClickHouse versions than the destination server.
+
+Users can export partitions from tables on servers with older ClickHouse versions to tables on servers with newer versions, enabling data migration and version upgrades.
+
+## Supported source part storage types
+
+### RQ.ClickHouse.ExportPartition.SourcePartStorage
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions regardless of the underlying storage type where the source parts are stored, including:
+* **Local Disks**: Parts stored on local filesystem
+* **S3/Object Storage**: Parts stored on S3 or S3-compatible object storage
+* **Encrypted Disks**: Parts stored on encrypted disks (disk-level encryption)
+* **Cached Disks**: Parts stored with filesystem cache enabled
+* **Remote Disks**: Parts stored on HDFS, Azure Blob Storage, or Google Cloud Storage
+* **Tiered Storage**: Parts stored across multiple storage tiers (hot/cold)
+* **Zero-Copy Replication Disks**: Parts stored with zero-copy replication enabled
+
+Users should be able to export partitions regardless of where the source data is physically stored, providing flexibility in storage configurations.
+
+## Storage policies and volumes
+
+### RQ.ClickHouse.ExportPartition.StoragePolicies
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from tables using different storage policies, where storage policies are composed of volumes which are composed of disks, including:
+* **JBOD Volumes**: Just a Bunch Of Disks volumes with multiple disks
+* **External Volumes**: Volumes using external storage systems
+* **Tiered Storage Policies**: Storage policies with multiple volumes for hot/cold data tiers
+* **Custom Storage Policies**: Any storage policy configuration composed of volumes and disks
+* Exporting all parts in a partition regardless of which volume or disk within the storage policy contains each part
+* Maintaining data integrity when exporting from parts stored on any volume or disk in the storage policy
+
+Users may have partitions with parts distributed across different storage tiers or volumes, and the export should handle all parts regardless of their storage location.
+
+## Supported destination table engines
+
+### RQ.ClickHouse.ExportPartition.DestinationEngines
+version: 1.0
+
+[ClickHouse] SHALL support exporting to destination tables that:
+* Support object storage engines including:
+ * `S3` - Amazon S3 and S3-compatible storage
+ * `StorageObjectStorage` - Generic object storage interface
+ * `HDFS` - Hadoop Distributed File System (with Hive partitioning)
+ * `Azure` - Microsoft Azure Blob Storage (with Hive partitioning)
+ * `GCS` - Google Cloud Storage (with Hive partitioning)
+
+Export partition is designed to move data from local or replicated storage to object storage systems for long-term storage, analytics, or data sharing purposes.
+
+## Temporary tables
+
+### RQ.ClickHouse.ExportPartition.TemporaryTable
+version: 1.0
+
+[ClickHouse] SHALL support exporting partitions from temporary ReplicatedMergeTree tables to destination object storage tables.
+
+For example,
+
+```sql
+CREATE TEMPORARY TABLE temp_table (p UInt64, k String, d UInt64)
+ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/temp_table', '{replica}')
+PARTITION BY p ORDER BY k;
+
+INSERT INTO temp_table VALUES (2020, 'key1', 100), (2020, 'key2', 200);
+
+ALTER TABLE temp_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+## Schema compatibility
+
+### RQ.ClickHouse.ExportPartition.SchemaCompatibility
+version: 1.0
+
+[ClickHouse] SHALL require source and destination tables to have compatible schemas for successful export operations:
+* Identical physical column schemas between source and destination
+* The same partition key expression in both tables
+* Compatible data types for all columns
+* Matching column order and names
+
+Schema compatibility ensures that exported data can be correctly read from the destination table without data loss or corruption.
+
+## Partition key types support
+
+### RQ.ClickHouse.ExportPartition.PartitionKeyTypes
+version: 1.0
+
+[ClickHouse] SHALL support export operations for tables with partition key types that are compatible with Hive partitioning, as shown in the following table:
+
+| Partition Key Type | Supported | Examples | Notes |
+|-------------------------|-----------|--------------------------------------------------------------------------|--------------------------------|
+| **Integer Types** | ✅ Yes | `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64` | All integer types supported |
+| **Date/DateTime Types** | ✅ Yes | `Date`, `Date32`, `DateTime`, `DateTime64` | All date/time types supported |
+| **String Types** | ✅ Yes | `String`, `FixedString` | All string types supported |
+| **No Partition Key** | ✅ Yes | Tables without `PARTITION BY` clause | Unpartitioned tables supported |
+
+[ClickHouse] SHALL automatically extract partition values from source parts and use them to create proper Hive partitioning structure in destination storage, but only for partition key types that are compatible with Hive partitioning requirements.
+
+[ClickHouse] SHALL require destination tables to support Hive partitioning, which limits the supported partition key types to Integer, Date/DateTime, and String types. Complex expressions that result in unsupported types are not supported for export operations.
+
+Hive partitioning is a standard way to organize data in object storage systems, making exported data compatible with various analytics tools and systems.
+
+## Partition content support
+
+### RQ.ClickHouse.ExportPartition.PartitionContent
+version: 1.0
+
+[ClickHouse] SHALL support export operations for partitions containing all valid MergeTree part types and their contents, including:
+
+| Part Type | Supported | Description | Special Features |
+|-------------------|-----------|--------------------------------------------------------------|--------------------------------|
+| **Wide Parts** | ✅ Yes | Data of each column stored in separate files with marks | Standard format for most parts |
+| **Compact Parts** | ✅ Yes | All column data stored in single file with single marks file | Optimized for small parts |
+
+[ClickHouse] SHALL export all parts within the specified partition, regardless of their type. The system SHALL automatically apply lightweight delete masks during export to ensure only non-deleted rows are exported, and SHALL maintain data integrity in the destination storage.
+
+Partitions may contain a mix of different part types, and the export must handle all of them correctly to ensure complete partition export.
+
+### RQ.ClickHouse.ExportPartition.SchemaChangeIsolation
+version: 1.0
+
+[ClickHouse] SHALL ensure exported partition data is isolated from subsequent schema changes by:
+* Preserving exported data exactly as it was at the time of export
+* Not being affected by schema changes (column drops, renames, type changes) that occur after export
+* Maintaining data integrity in destination storage regardless of mutations applied to the source table after export
+* Ensuring exported data reflects the source table state at the time of export, not the current state
+
+Once a partition is exported, the exported data should remain stable and not be affected by future changes to the source table schema.
+
+### RQ.ClickHouse.ExportPartition.LargePartitions
+version: 1.0
+
+[ClickHouse] SHALL support exporting large partitions by:
+* Handling partitions with large numbers of parts (e.g., hundreds or thousands of parts)
+* Processing partitions with large numbers of rows (e.g., billions of rows)
+* Processing large data volumes efficiently during export
+* Maintaining data integrity when exporting large partitions
+* Completing export operations successfully regardless of partition size
+* Allowing export operations to continue over extended periods of time for very large partitions
+
+Production systems often have partitions containing very large amounts of data, and the export must handle these efficiently without timeouts or memory issues.
+
+### RQ.ClickHouse.ExportPartition.Corrupted
+version: 1.0
+
+[ClickHouse] SHALL output an error and prevent export operations from proceeding when trying to export a partition that contains corrupted parts in the source table.
+
+The system SHALL detect corruption in partitions containing compact parts, wide parts, or mixed part types.
+
+## Export operation failure handling
+
+### RQ.ClickHouse.ExportPartition.RetryMechanism
+version: 1.0
+
+[ClickHouse] SHALL automatically retry failed part exports within a partition up to a configurable maximum retry count. If all retry attempts are exhausted for a part, the entire partition export operation SHALL be marked as failed.
+
+Unlike single-part exports, partition exports involve multiple parts and may take significant time. Retry mechanisms ensure that temporary failures don't require restarting the entire export operation.
+
+### RQ.ClickHouse.ExportPartition.Settings.MaxRetries
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_max_retries` setting that controls the maximum number of retries for exporting a merge tree part in an export partition task. The default value SHALL be `3`.
+
+This setting allows users to control how many times the system will retry exporting a part before marking it as failed.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_max_retries = 5
+```
+
+### RQ.ClickHouse.ExportPartition.ResumeAfterFailure
+version: 1.0
+
+[ClickHouse] SHALL allow export operations to resume after node failures or restarts. The system SHALL track which parts have been successfully exported and SHALL not re-export parts that were already successfully exported.
+
+### RQ.ClickHouse.ExportPartition.PartialProgress
+version: 1.0
+
+[ClickHouse] SHALL allow export operations to make partial progress, with successfully exported parts remaining in the destination even if other parts fail. Users SHALL be able to see which parts have been successfully exported and which parts have failed.
+
+For example, users can query the export status to see partial progress:
+
+```sql
+SELECT source_table, destination_table, partition_id, status,
+ parts_total, parts_processed, parts_failed
+FROM system.replicated_partition_exports
+WHERE partition_id = '2020'
+```
+
+### RQ.ClickHouse.ExportPartition.Cleanup
+version: 1.0
+
+[ClickHouse] SHALL automatically clean up failed or completed export operations after a configurable TTL period.
+
+### RQ.ClickHouse.ExportPartition.Settings.ManifestTTL
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_manifest_ttl` setting that determines how long the export manifest will be retained. This setting prevents the same partition from being exported twice to the same destination within the TTL period. The default value SHALL be `180` seconds.
+
+This setting only affects completed export operations and does not delete in-progress tasks. It allows users to control how long export history is maintained to prevent duplicate exports.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_manifest_ttl = 360
+```
+
+## Network resilience
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.PacketIssues
+version: 1.0
+
+[ClickHouse] SHALL handle network packet issues during export operations by:
+* Tolerating packet delay without data corruption or loss
+* Handling packet loss and retransmitting data as needed
+* Detecting and handling packet corruption to ensure data integrity
+* Managing packet duplication without data duplication in destination
+* Handling packet reordering to maintain correct data sequence
+* Operating correctly under packet rate limiting constraints
+* Completing exports successfully despite network impairments
+
+Network issues are common in distributed systems, and export operations must be resilient to ensure data integrity.
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.DestinationInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle destination storage interruptions during export operations by:
+* Detecting when destination storage becomes unavailable during export
+* Retrying failed part exports when destination storage becomes available again
+* Logging failed exports in the `system.events` table with appropriate counters
+* Not leaving partial or corrupted data in destination storage when exports fail due to destination unavailability
+* Allowing exports to complete successfully once destination storage becomes available again
+* Resuming export operations from the last successfully exported part
+
+Destination storage systems may experience temporary outages, and the export should automatically recover when service is restored.
+
+### RQ.ClickHouse.ExportPartition.NetworkResilience.NodeInterruption
+version: 1.0
+
+[ClickHouse] SHALL handle ClickHouse node interruptions during export operations by:
+* Allowing export operations to resume after node restart without data loss or duplication
+* Allowing other replicas to continue or complete export operations if a node fails
+* Not leaving partial or corrupted data in destination storage when node restarts occur
+* With safe shutdown, ensuring exports complete successfully before node shutdown when possible
+* With unsafe shutdown, allowing export operations to resume from the last checkpoint after node restart
+* Maintaining data integrity in destination storage regardless of node interruption type
+* Ensuring that parts already exported are not re-exported after node restart
+
+Node failures are common in distributed systems, and export operations must be able to recover and continue without data loss or duplication.
+
+## Export operation restrictions
+
+### Preventing same table exports
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.SameTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting partitions to the same table as the source by:
+* Validating that source and destination table identifiers are different
+* Throwing a `BAD_ARGUMENTS` exception with message "Exporting to the same table is not allowed" when source and destination are identical
+* Performing this validation before any export processing begins
+
+Exporting to the same table would be redundant and could cause data duplication or conflicts.
+
+For example, the following command SHALL output an error:
+
+```sql
+ALTER TABLE my_table
+EXPORT PARTITION ID '2020'
+TO TABLE my_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Destination table compatibility
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.DestinationSupport
+version: 1.0
+
+[ClickHouse] SHALL validate destination table compatibility by:
+
+* Checking that the destination storage supports importing MergeTree parts
+* Verifying that the destination uses Hive partitioning strategy (`partition_strategy = 'hive'`)
+* Throwing a `NOT_IMPLEMENTED` exception with message "Destination storage {} does not support MergeTree parts or uses unsupported partitioning" when requirements are not met
+* Performing this validation during the initial export setup phase
+
+The destination must support the format and partitioning strategy required for exported data.
+
+### Local table restriction
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.LocalTable
+version: 1.0
+
+[ClickHouse] SHALL prevent exporting partitions to local MergeTree tables by:
+* Rejecting export operations where the destination table uses a MergeTree engine
+* Throwing a `NOT_IMPLEMENTED` exception (error code 48) with message "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning" when attempting to export to a local table
+* Performing this validation during the initial export setup phase
+
+Export partition is designed to move data to object storage, not to local MergeTree tables.
+
+For example, if `local_table` is a MergeTree table, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE local_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Partition key compatibility
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.PartitionKey
+version: 1.0
+
+[ClickHouse] SHALL validate that source and destination tables have the same partition key expression by:
+* Checking that the partition key expression matches between source and destination tables
+* Throwing a `BAD_ARGUMENTS` exception (error code 36) with message "Tables have different partition key" when partition keys differ
+* Performing this validation during the initial export setup phase
+
+Matching partition keys ensure that exported data is organized correctly in the destination storage.
+
+For example, if `source_table` is partitioned by `toYYYYMM(date)` and `destination_table` is partitioned by `toYYYYMMDD(date)`, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+### Source partition availability
+
+#### RQ.ClickHouse.ExportPartition.Restrictions.SourcePartition
+version: 1.0
+
+[ClickHouse] SHALL validate source partition availability by:
+* Checking that the specified partition ID exists in the source table
+* Verifying that the partition contains at least one active part (not detached or missing)
+* Throwing an exception with an appropriate error message when the partition is not found or is empty
+* Performing this validation before any export processing begins
+
+The system must verify that the partition exists and contains data before attempting to export it.
+
+For example, if partition ID '2025' does not exist in `source_table`, the following command SHALL output an error:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2025'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1
+```
+
+## Export operation concurrency
+
+### RQ.ClickHouse.ExportPartition.Concurrency
+version: 1.0
+
+[ClickHouse] SHALL support concurrent export operations by:
+* Allowing multiple partition exports to run simultaneously without interference
+* Supporting concurrent exports of different partitions to different destinations
+* Preventing concurrent exports of the same partition to the same destination
+* Allowing different replicas to export different parts of the same partition concurrently
+* Maintaining separate progress tracking for each concurrent operation
+
+Multiple users may want to export different partitions simultaneously, and the system must coordinate these operations to prevent conflicts while maximizing parallelism.
+
+## Export operation idempotency
+
+### RQ.ClickHouse.ExportPartition.Idempotency
+version: 1.0
+
+[ClickHouse] SHALL handle duplicate export operations by:
+* Preventing duplicate data from being exported when the same partition is exported multiple times to the same destination
+* Detecting when a partition export is already in progress or completed
+* Detecting when an export operation attempts to export a partition that already exists in the destination
+* Logging duplicate export attempts in the `system.events` table with appropriate counters
+* Ensuring that destination data matches source data without duplication when the same partition is exported multiple times
+* Allowing users to force re-export of a partition if needed (e.g., after TTL expiration or manual cleanup)
+
+Users may accidentally trigger the same export multiple times, and the system should prevent duplicate data while allowing legitimate re-exports when needed.
+
+### RQ.ClickHouse.ExportPartition.Settings.ForceExport
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_partition_force_export` setting that allows users to ignore existing partition export entries and force a new export operation. The default value SHALL be `false` (turned off).
+
+When set to `true`, this setting allows users to overwrite existing export entries and force re-export of a partition, even if a previous export operation exists for the same partition and destination.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_partition_force_export = 1
+```
+
+## Export operation logging
+
+### RQ.ClickHouse.ExportPartition.Logging
+version: 1.0
+
+[ClickHouse] SHALL provide detailed logging for export operations by:
+* Logging all export operations (both successful and failed) with timestamps and details
+* Recording the specific partition ID in the `system.part_log` table for all operations
+* Logging export events in the `system.events` table, including:
+ * `PartsExports` - Number of successful part exports (within partitions)
+ * `PartsExportFailures` - Number of failed part exports
+ * `PartsExportDuplicated` - Number of part exports that failed because target already exists
+* Writing operation information to the `system.part_log` table with `event_type` set to `EXPORT_PARTITION`
+* Providing sufficient detail for monitoring and troubleshooting export operations
+* Logging per-part export status within partition exports
+
+Detailed logging helps users monitor export progress, troubleshoot issues, and audit export operations.
+
+For example, users can query export logs:
+
+```sql
+SELECT event_time, event_type, table, partition, rows, bytes_read, bytes_written
+FROM system.part_log
+WHERE event_type = 'EXPORT_PARTITION'
+ORDER BY event_time DESC
+```
+
+## Monitoring export operations
+
+### RQ.ClickHouse.ExportPartition.SystemTables.Exports
+version: 1.0
+
+[ClickHouse] SHALL provide a `system.replicated_partition_exports` table that allows users to monitor active partition export operations with at least the following columns:
+* `source_table` - source table identifier
+* `destination_table` - destination table identifier
+* `partition_id` - the partition ID being exported
+* `status` - current status of the export operation (e.g., PENDING, IN_PROGRESS, COMPLETED, FAILED)
+* `parts_total` - total number of parts in the partition
+* `parts_processed` - number of parts successfully exported
+* `parts_failed` - number of parts that failed to export
+* `create_time` - when the export operation was created
+* `update_time` - last update time of the export operation
+
+The table SHALL track export operations before they complete and SHALL show completed or failed exports until they are cleaned up (based on TTL).
+
+Users need visibility into export operations to monitor progress, identify issues, and understand export status across the cluster.
+
+For example,
+
+```sql
+SELECT source_table, destination_table, partition_id, status,
+ parts_total, parts_processed, parts_failed, create_time, update_time
+FROM system.replicated_partition_exports
+WHERE status = 'IN_PROGRESS'
+```
+
+## Enabling export functionality
+
+### RQ.ClickHouse.ExportPartition.Settings.AllowExperimental
+version: 1.0
+
+[ClickHouse] SHALL support the `allow_experimental_export_merge_tree_part` setting that SHALL gate the experimental export partition functionality, which SHALL be set to `1` to enable `ALTER TABLE ... EXPORT PARTITION ID ...` commands. The default value SHALL be `0` (turned off).
+
+This setting allows administrators to control access to experimental functionality and ensures users are aware they are using a feature that may change.
+
+### RQ.ClickHouse.ExportPartition.Settings.AllowExperimental.Disabled
+version: 1.0
+
+[ClickHouse] SHALL prevent export partition operations when `allow_experimental_export_merge_tree_part` is set to `0` (turned off). When the setting is `0`, attempting to execute `ALTER TABLE ... EXPORT PARTITION ID ...` commands SHALL result in an error indicating that the experimental feature is not enabled.
+
+For example, the following command SHALL output an error when the setting is `0`:
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+```
+
+## Handling file conflicts during export
+
+### RQ.ClickHouse.ExportPartition.Settings.OverwriteFile
+version: 1.0
+
+[ClickHouse] SHALL support the `export_merge_tree_part_overwrite_file_if_exists` setting that controls whether to overwrite files if they already exist when exporting a partition. The default value SHALL be `0` (turned off), meaning exports will fail if files already exist in the destination.
+
+This setting allows users to control whether to overwrite existing data in the destination, providing safety by default while allowing overwrites when needed.
+
+For example,
+
+```sql
+ALTER TABLE source_table
+EXPORT PARTITION ID '2020'
+TO TABLE destination_table
+SETTINGS allow_experimental_export_merge_tree_part = 1,
+ export_merge_tree_part_overwrite_file_if_exists = 1
+```
+
+## Export operation configuration
+
+### RQ.ClickHouse.ExportPartition.ParallelFormatting
+version: 1.0
+
+[ClickHouse] SHALL support parallel formatting for export operations by:
+* Automatically enabling parallel formatting for large export operations to improve performance
+* Using the `output_format_parallel_formatting` setting to control parallel formatting behavior
+* Optimizing data processing based on export size and system resources
+* Providing consistent formatting performance across different export scenarios
+* Allowing parallel processing of multiple parts within a partition when possible
+
+Parallel formatting improves export performance, especially for large partitions with many parts.
+
+## Controlling export performance
+
+### RQ.ClickHouse.ExportPartition.ServerSettings.MaxBandwidth
+version: 1.0
+
+[ClickHouse] SHALL support the `max_exports_bandwidth_for_server` server setting to limit the maximum read speed of all exports on the server in bytes per second, with `0` meaning unlimited bandwidth. The default value SHALL be `0`. This is a server-level setting configured in the server configuration file.
+
+Administrators need to control export bandwidth to avoid impacting other operations on the server.
+
+### RQ.ClickHouse.ExportPartition.ServerSettings.BackgroundMovePoolSize
+version: 1.0
+
+[ClickHouse] SHALL support the `background_move_pool_size` server setting to control the maximum number of threads that will be used for executing export operations in the background. The default value SHALL be `8`. This is a server-level setting configured in the server configuration file.
+
+This setting allows administrators to balance export performance with other system operations.
+
+### RQ.ClickHouse.ExportPartition.Metrics.Export
+version: 1.0
+
+[ClickHouse] SHALL provide the `Export` current metric in `system.metrics` table that tracks the number of currently executing partition exports.
+
+This metric helps monitor system load from export operations.
+
+## Export operation security
+
+### RQ.ClickHouse.ExportPartition.Security.RBAC
+version: 1.0
+
+[ClickHouse] SHALL enforce role-based access control (RBAC) for export operations. Users must have the following privileges to perform export operations:
+* **Source Table**: `SELECT` privilege on the source table to read data parts
+* **Destination Table**: `INSERT` privilege on the destination table to write exported data
+* **Database Access**: `SHOW` privilege on both source and destination databases
+* **System Tables**: `SELECT` privilege on `system.tables` and `system.replicated_partition_exports` to validate table existence and monitor exports
+
+Export operations move potentially sensitive data, and proper access controls ensure only authorized users can export partitions.
+
+### RQ.ClickHouse.ExportPartition.Security.DataEncryption
+version: 1.0
+
+[ClickHouse] SHALL encrypt all data in transit to destination storage using TLS/SSL during export operations.
+
+Data encryption protects sensitive information from being intercepted or accessed during transmission to destination storage.
+
+### RQ.ClickHouse.ExportPartition.Security.Network
+version: 1.0
+
+[ClickHouse] SHALL use secure connections to destination storage during export operations. For S3-compatible storage, connections must use HTTPS. For other storage types, secure protocols appropriate to the storage system must be used.
+
+Secure network connections prevent unauthorized access and ensure data integrity during export operations.
+
+### RQ.ClickHouse.ExportPartition.Security.CredentialManagement
+version: 1.0
+
+[ClickHouse] SHALL use secure credential storage for export operations and SHALL avoid exposing credentials in logs or error messages.
+
+Proper credential management prevents unauthorized access to destination storage systems and protects sensitive authentication information.
+
+
+[ClickHouse]: https://clickhouse.com
+""",
+)
diff --git a/s3/s3_env/clickhouse-service.yml b/s3/s3_env/clickhouse-service.yml
old mode 100644
new mode 100755
index f3cc6cc7d..c766d2085
--- a/s3/s3_env/clickhouse-service.yml
+++ b/s3/s3_env/clickhouse-service.yml
@@ -22,3 +22,5 @@ services:
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/system_unfreeze.xml:/etc/clickhouse-server/config.d/system_unfreeze.xml"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/output_format_json_quote_64bit_integers.xml:/etc/clickhouse-server/users.d/output_format_json_quote_64bit_integers.xml"
+ cap_add:
+ - NET_ADMIN
\ No newline at end of file
diff --git a/s3/s3_env_arm64/clickhouse-service.yml b/s3/s3_env_arm64/clickhouse-service.yml
old mode 100644
new mode 100755
index f1feb9ad8..f5f2dfeef
--- a/s3/s3_env_arm64/clickhouse-service.yml
+++ b/s3/s3_env_arm64/clickhouse-service.yml
@@ -22,3 +22,5 @@ services:
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/system_unfreeze.xml:/etc/clickhouse-server/config.d/system_unfreeze.xml"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/output_format_json_quote_64bit_integers.xml:/etc/clickhouse-server/users.d/output_format_json_quote_64bit_integers.xml"
+ cap_add:
+ - NET_ADMIN
\ No newline at end of file
diff --git a/s3/tests/export_part/clusters_nodes.py b/s3/tests/export_part/clusters_nodes.py
new file mode 100644
index 000000000..cfd2ed209
--- /dev/null
+++ b/s3/tests/export_part/clusters_nodes.py
@@ -0,0 +1,79 @@
+from itertools import combinations
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.queries import *
+from alter.table.replace_partition.common import create_partitions_with_random_uint64
+from s3.requirements.export_part import *
+
+
+@TestScenario
+def different_nodes_same_destination(self, cluster, node1, node2):
+ """Test export part from different nodes to same S3 destination in a given cluster."""
+
+ with Given("I create an empty source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ cluster=cluster,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3", create_new_bucket=True, cluster=cluster
+ )
+
+ with And("I populate the source tables on both nodes"):
+ create_partitions_with_random_uint64(table_name="source", node=node1)
+ create_partitions_with_random_uint64(table_name="source", node=node2)
+
+ with When("I export parts to the S3 table from both nodes"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=node1,
+ )
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=node2,
+ )
+
+ with And("I read data from all tables on both nodes"):
+ source_data1 = select_all_ordered(table_name="source", node=node1)
+ source_data2 = select_all_ordered(table_name="source", node=node2)
+ destination_data1 = select_all_ordered(table_name=s3_table_name, node=node1)
+ destination_data2 = select_all_ordered(table_name=s3_table_name, node=node2)
+
+ with Then(
+ "Destination data should be comprised of data from both sources, and identical on both nodes"
+ ):
+ assert set(destination_data1) == set(source_data1) | set(source_data2), error()
+ assert set(destination_data2) == set(source_data1) | set(source_data2), error()
+
+
+@TestFeature
+@Requirements(RQ_ClickHouse_ExportPart_ClustersNodes("1.0"))
+@Name("clusters and nodes")
+def feature(self):
+ """Check functionality of exporting data parts to S3 storage from different clusters and nodes."""
+
+ clusters = [
+ "sharded_cluster",
+ "replicated_cluster",
+ "one_shard_cluster",
+ "sharded_cluster12",
+ "one_shard_cluster12",
+ "sharded_cluster23",
+ "one_shard_cluster23",
+ ]
+
+ for cluster in clusters:
+ with Given(f"I get nodes for cluster {cluster}"):
+ node_names = get_cluster_nodes(cluster=cluster)
+
+ for node1_name, node2_name in combinations(node_names, 2):
+ node1 = self.context.cluster.node(node1_name)
+ node2 = self.context.cluster.node(node2_name)
+ different_nodes_same_destination(cluster=cluster, node1=node1, node2=node2)
diff --git a/s3/tests/export_part/concurrency_networks.py b/s3/tests/export_part/concurrency_networks.py
new file mode 100644
index 000000000..31f731ee0
--- /dev/null
+++ b/s3/tests/export_part/concurrency_networks.py
@@ -0,0 +1,456 @@
+from testflows.core import *
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.queries import *
+from s3.requirements.export_part import *
+from alter.stress.tests.tc_netem import *
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Concurrency("1.0"))
+def concurrent_export(self, num_tables):
+ """Check concurrent exports from different sources to the same S3 table."""
+
+ with Given(f"I create {num_tables} populated source tables and an empty S3 table"):
+ source_tables = []
+ for i in range(num_tables):
+ source_tables.append(
+ partitioned_merge_tree_table(
+ table_name=f"source_{getuid()}",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts from all sources concurrently to the S3 table"):
+ for i in range(num_tables):
+ Step(test=export_parts, parallel=True)(
+ source_table=source_tables[i],
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ join()
+
+ with And("I read data from all tables"):
+ source_data = []
+ for i in range(num_tables):
+ data = select_all_ordered(
+ table_name=source_tables[i], node=self.context.node
+ )
+ source_data.extend(data)
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+
+ with Then("All data should be present in the S3 table"):
+ assert set(source_data) == set(destination_data), error()
+
+ with And("Exports should have run concurrently"):
+ verify_export_concurrency(node=self.context.node, source_tables=source_tables)
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_delay(self, delay_ms):
+ """Check that exports work correctly with packet delay."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply a packet delay"):
+ network_packet_delay(node=self.context.node, delay_ms=delay_ms)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_loss(self, percent_loss):
+ """Check that exports work correctly with packet loss."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet loss"):
+ network_packet_loss(node=self.context.node, percent_loss=percent_loss)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_loss_gemodel(self, interruption_probability, recovery_probability):
+ """Check that exports work correctly with packet loss using the GE model."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet loss using the GE model"):
+ network_packet_loss_gemodel(
+ node=self.context.node,
+ interruption_probability=interruption_probability,
+ recovery_probability=recovery_probability,
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_corruption(self, percent_corrupt):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet corruption"):
+ network_packet_corruption(
+ node=self.context.node, percent_corrupt=percent_corrupt
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_duplication(self, percent_duplicated):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet duplication"):
+ network_packet_duplication(
+ node=self.context.node, percent_duplicated=percent_duplicated
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_reordering(self, delay_ms, percent_reordered):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet reordering"):
+ network_packet_reordering(
+ node=self.context.node,
+ delay_ms=delay_ms,
+ percent_reordered=percent_reordered,
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_PacketIssues("1.0"))
+def packet_rate_limit(self, rate_mbit):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet rate limit"):
+ network_packet_rate_limit(node=self.context.node, rate_mbit=rate_mbit)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Concurrency("1.0"))
+def concurrent_insert(self):
+ """Check that exports work correctly with concurrent inserts of source data."""
+
+ with Given("I create an empty source and S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When(
+ "I insert data and export it in parallel",
+ description="""
+ 5 partitions with 1 part each are inserted.
+ The export is queued in parallel and usually behaves by exporting
+ a snapshot of the source data, often getting just the first partition
+ which means the export happens right after the first INSERT query completes.
+ """,
+ ):
+ Step(test=create_partitions_with_random_uint64, parallel=True)(
+ table_name="source",
+ number_of_partitions=5,
+ number_of_parts=1,
+ )
+ Step(test=export_parts, parallel=True)(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ join()
+
+ with Then("Destination data should be a subset of source data"):
+ source_data = select_all_ordered(table_name="source", node=self.context.node)
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+ assert set(source_data) >= set(destination_data), error()
+
+ with And("Inserts should have completed successfully"):
+ assert len(source_data) == 15, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_DestinationInterruption("1.0"))
+def minio_network_interruption(self, number_of_values=3, signal="KILL"):
+ """Check that restarting MinIO while exporting parts inbetween works correctly."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ number_of_values=number_of_values,
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I stop MinIO"):
+ kill_minio(signal=signal)
+
+ with When("I read export events"):
+ initial_events = get_export_events(node=self.context.node)
+
+ with And("I export data"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I start MinIO"):
+ start_minio()
+
+ with Then("Destination data should be a subset of source data"):
+ source_data = select_all_ordered(table_name="source", node=self.context.node)
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+ assert set(source_data) >= set(destination_data), error()
+
+ with And("Failed exports should be logged in the system.events table"):
+ final_events = get_export_events(node=self.context.node)
+ assert (
+ final_events["PartsExportFailures"] - initial_events["PartsExportFailures"]
+ == (len(source_data) - len(destination_data)) / number_of_values
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_NetworkResilience_NodeInterruption("1.0"))
+def clickhouse_network_interruption(self, safe=False):
+ """Check that exports work correctly with a clickhouse network outage."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I get parts before the interruption"):
+ parts = get_parts(table_name="source", node=self.context.node)
+
+ with When("I queue exports and restart the node in parallel"):
+ Step(test=export_parts, parallel=True)(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ parts=parts,
+ )
+ self.context.node.restart(safe=safe)
+ join()
+
+ if safe:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+ else:
+ with Then("Destination data should be a subset of source data"):
+ source_data = select_all_ordered(
+ table_name="source", node=self.context.node
+ )
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+ assert set(source_data) >= set(destination_data), error()
+
+
+@TestFeature
+@Name("concurrency and networks")
+def feature(self):
+ """Check that exports work correctly with concurrency and various network conditions."""
+
+ # TODO corruption (bit flipping)
+
+ Scenario(test=concurrent_export)(num_tables=5)
+ Scenario(test=packet_delay)(delay_ms=100)
+ Scenario(test=packet_loss)(percent_loss=50)
+ Scenario(test=packet_loss_gemodel)(
+ interruption_probability=40, recovery_probability=70
+ )
+ Scenario(test=packet_corruption)(percent_corrupt=50)
+ Scenario(test=packet_duplication)(percent_duplicated=50)
+ Scenario(test=packet_reordering)(delay_ms=100, percent_reordered=90)
+ Scenario(test=packet_rate_limit)(rate_mbit=0.05)
+ Scenario(run=concurrent_insert)
+ Scenario(test=minio_network_interruption)(signal="TERM")
+ Scenario(test=minio_network_interruption)(signal="KILL")
+ Scenario(test=clickhouse_network_interruption)(safe=True)
+ Scenario(test=clickhouse_network_interruption)(safe=False)
diff --git a/s3/tests/export_part/datatypes.py b/s3/tests/export_part/datatypes.py
new file mode 100644
index 000000000..a7f1af231
--- /dev/null
+++ b/s3/tests/export_part/datatypes.py
@@ -0,0 +1,122 @@
+from testflows.core import *
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.queries import *
+from helpers.common import getuid
+from s3.requirements.export_part import *
+
+
+@TestStep(When)
+def insert_all_datatypes(self, table_name, rows_per_part=1, num_parts=1, node=None):
+ """Insert all datatypes into a MergeTree table."""
+
+ if node is None:
+ node = self.context.node
+
+ for part in range(num_parts):
+ node.query(
+ f"INSERT INTO {table_name} (int8, int16, int32, int64, uint8, uint16, uint32, uint64, date, date32, datetime, datetime64, string, fixedstring) SELECT 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, '13', '14' FROM numbers({rows_per_part})"
+ )
+
+
+@TestStep(Given)
+def create_merge_tree_all_valid_partition_key_types(
+ self, column_name, cluster=None, node=None, rows_per_part=1
+):
+ """Create a MergeTree table with all valid partition key types and both wide and compact parts."""
+
+ if node is None:
+ node = self.context.node
+
+ with By("creating a MergeTree table with all data types"):
+ table_name = f"table_{getuid()}"
+ create_merge_tree_table(
+ table_name=table_name,
+ columns=valid_partition_key_types_columns(),
+ partition_by=column_name,
+ cluster=cluster,
+ stop_merges=True,
+ query_settings=f"min_rows_for_wide_part=10",
+ )
+
+ with And("I insert compact and wide parts into the table"):
+ insert_all_datatypes(
+ table_name=table_name,
+ rows_per_part=rows_per_part,
+ num_parts=self.context.num_parts,
+ node=node,
+ )
+
+ return table_name
+
+
+@TestCheck
+def valid_partition_key_table(self, partition_key_type, rows_per_part=1):
+ """Check exporting to a source table with specified valid partition key type and rows."""
+
+ with Given(
+ f"I create a source table with valid partition key type {partition_key_type} and empty S3 table"
+ ):
+ table_name = create_merge_tree_all_valid_partition_key_types(
+ column_name=partition_key_type,
+ rows_per_part=rows_per_part,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=valid_partition_key_types_columns(),
+ partition_by=partition_key_type,
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=table_name,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read data from both tables"):
+ source_data = select_all_ordered(
+ table_name=table_name, node=self.context.node, order_by=partition_key_type
+ )
+ destination_data = select_all_ordered(
+ table_name=s3_table_name,
+ node=self.context.node,
+ order_by=partition_key_type,
+ )
+
+ with Then("They should be the same"):
+ assert source_data == destination_data, error()
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+def valid_partition_key_types_compact(self):
+ """Check that all partition key data types are supported when exporting compact parts."""
+
+ key_types = [datatype["name"] for datatype in valid_partition_key_types_columns()]
+ valid_partition_key_table(partition_key_type=either(*key_types), rows_per_part=1)
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+def valid_partition_key_types_wide(self):
+ """Check that all partition key data types are supported when exporting wide parts."""
+
+ key_types = [datatype["name"] for datatype in valid_partition_key_types_columns()]
+ valid_partition_key_table(partition_key_type=either(*key_types), rows_per_part=100)
+
+
+@TestFeature
+@Name("datatypes")
+@Requirements(
+ RQ_ClickHouse_ExportPart_PartitionKeyTypes("1.0"),
+ RQ_ClickHouse_ExportPart_PartTypes("1.0"),
+)
+def feature(self, num_parts=10):
+ """Check that all data types are supported when exporting parts."""
+
+ self.context.num_parts = num_parts
+
+ Scenario(run=valid_partition_key_types_compact)
+ Scenario(run=valid_partition_key_types_wide)
diff --git a/s3/tests/export_part/engines_volumes.py b/s3/tests/export_part/engines_volumes.py
new file mode 100644
index 000000000..150f409f8
--- /dev/null
+++ b/s3/tests/export_part/engines_volumes.py
@@ -0,0 +1,156 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from s3.requirements.export_part import *
+from s3.tests.common import s3_storage
+from helpers.queries import *
+from helpers.common import getuid
+from testflows.combinatorics import product
+
+
+@TestCheck
+def configured_table(self, table_engine, number_of_partitions, number_of_parts):
+ """Test a specific combination of table engine, number of partitions, and number of parts."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ source_table = table_engine(
+ table_name=f"source_{getuid()}",
+ partition_by="p",
+ stop_merges=True,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ columns=default_columns(simple=False, partition_key_type="Int8"),
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=default_columns(simple=False, partition_key_type="Int8"),
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Source and destination tables should match"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+@Requirements(RQ_ClickHouse_ExportPart_SourceEngines("1.0"))
+def table_combos(self):
+ """Test various combinations of table engines, number of partitions, and number of parts."""
+
+ tables = [
+ partitioned_merge_tree_table,
+ partitioned_replacing_merge_tree_table,
+ partitioned_summing_merge_tree_table,
+ partitioned_collapsing_merge_tree_table,
+ partitioned_versioned_collapsing_merge_tree_table,
+ partitioned_aggregating_merge_tree_table,
+ partitioned_graphite_merge_tree_table,
+ ]
+ number_of_partitions = [5] if not self.context.stress else [1, 5, 10]
+ number_of_parts = [1] if not self.context.stress else [1, 5, 10]
+
+ combinations = product(tables, number_of_partitions, number_of_parts)
+
+ with Pool(16) as executor:
+ for table_engine, number_of_partitions, number_of_parts in combinations:
+ Combination(
+ name=f"{table_engine.__name__} partitions={number_of_partitions} parts={number_of_parts}",
+ test=configured_table,
+ executor=executor,
+ parallel=True,
+ )(
+ table_engine=table_engine,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+ join()
+
+
+@TestCheck
+def configured_volume(self, volume):
+ """Test a specific combination of volume."""
+
+ with Given(f"I create an empty source table on volume {volume} and empty S3 table"):
+ source_table = partitioned_merge_tree_table(
+ table_name=f"source_{getuid()}",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ query_settings=f"storage_policy = '{volume}'",
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I populate the source table with parts exceeding 2KB each"):
+ create_partitions_with_random_uint64(
+ table_name=source_table,
+ node=self.context.node,
+ number_of_values=500,
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Source and destination tables should match"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+@Requirements(RQ_ClickHouse_ExportPart_StoragePolicies("1.0"))
+def volume_combos(self):
+ """Test exporting to various storage policies."""
+
+ volumes = [
+ "jbod1",
+ "jbod2",
+ "jbod3",
+ "jbod4",
+ "external",
+ "external2",
+ "tiered_storage",
+ ]
+ combinations = product(volumes)
+
+ with Pool(16) as executor:
+ for (volume,) in combinations:
+ Combination(
+ name=f"volume={volume}",
+ test=configured_volume,
+ executor=executor,
+ parallel=True,
+ )(
+ volume=volume,
+ )
+ join()
+
+
+@TestFeature
+@Name("engines and volumes")
+def feature(self):
+ """Check exporting parts to S3 storage with different table engines and volumes."""
+
+ # TODO replicated merge tree tables (all types)
+
+ with Given("I set up MinIO storage configuration"):
+ minio_storage_configuration(restart=True)
+
+ Scenario(run=table_combos)
+ Scenario(run=volume_combos)
diff --git a/s3/tests/export_part/error_handling.py b/s3/tests/export_part/error_handling.py
new file mode 100644
index 000000000..f2160696d
--- /dev/null
+++ b/s3/tests/export_part/error_handling.py
@@ -0,0 +1,173 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.queries import *
+from s3.requirements.export_part import *
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Restrictions_SourcePart("1.0"))
+def invalid_part_name(self):
+ """Check that exporting a non-existent part returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I create an invalid part name"):
+ invalid_part_name = "in_va_lid_part"
+
+ with When("I try to export the invalid part"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ parts=[invalid_part_name],
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to the invalid part name"):
+ assert results[0].exitcode == 233, error()
+ assert (
+ f"Unexpected part name: {invalid_part_name}" in results[0].output
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Restrictions_SameTable("1.0"))
+def same_table(self):
+ """Check exporting parts where source and destination tables are the same."""
+
+ with Given("I create a populated source table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+
+ with When("I try to export parts to itself"):
+ results = export_parts(
+ source_table="source",
+ destination_table="source",
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to same table exports"):
+ assert results[0].exitcode == 36, error()
+ assert (
+ "Exporting to the same table is not allowed" in results[0].output
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Restrictions_LocalTable("1.0"))
+def local_table(self):
+ """Test exporting parts to a local table."""
+
+ with Given("I create a populated source table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+
+ with And("I create an empty local table"):
+ partitioned_merge_tree_table(
+ table_name="destination",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ )
+
+ with When("I export parts to the local table"):
+ results = export_parts(
+ source_table="source",
+ destination_table="destination",
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to local table exports"):
+ assert results[0].exitcode == 48, error()
+ assert (
+ "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning"
+ in results[0].output
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Settings_AllowExperimental("1.0"))
+def disable_export_setting(self):
+ """Check that exporting parts without the export setting set returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I try to export the parts with the export setting disabled"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ inline_settings=[("allow_experimental_export_merge_tree_part", 0)],
+ )
+
+ with Then("I should see an error related to the export setting"):
+ assert results[0].exitcode == 88, error()
+ assert "Exporting merge tree part is experimental" in results[0].output, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Restrictions_PartitionKey("1.0"))
+def different_partition_key(self):
+ """Check exporting parts with a different partition key returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="i",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I try to export the parts"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to the different partition key"):
+ assert results[0].exitcode == 36, error()
+ assert "Tables have different partition key" in results[0].output, error()
+
+
+@TestFeature
+@Name("error handling")
+@Requirements(RQ_ClickHouse_ExportPart_FailureHandling("1.0"))
+def feature(self):
+ """Check correct error handling when exporting parts."""
+
+ Scenario(run=invalid_part_name)
+ Scenario(run=same_table)
+ Scenario(run=local_table)
+ Scenario(run=disable_export_setting)
+ Scenario(run=different_partition_key)
diff --git a/s3/tests/export_part/feature.py b/s3/tests/export_part/feature.py
new file mode 100644
index 000000000..c03045ffd
--- /dev/null
+++ b/s3/tests/export_part/feature.py
@@ -0,0 +1,26 @@
+from testflows.core import *
+from s3.requirements.export_part import *
+
+
+@TestFeature
+@Specifications(
+ SRS_015_ClickHouse_Export_Part_to_S3,
+)
+@Requirements(
+ RQ_ClickHouse_ExportPart_S3("1.0"),
+)
+@Name("export part")
+def minio(self, uri, bucket_prefix):
+ """Run features from the export parts suite."""
+
+ self.context.uri_base = uri
+ self.context.bucket_prefix = bucket_prefix
+ self.context.default_settings = [("allow_experimental_export_merge_tree_part", 1)]
+
+ Feature(run=load("s3.tests.export_part.sanity", "feature"))
+ Feature(run=load("s3.tests.export_part.error_handling", "feature"))
+ Feature(run=load("s3.tests.export_part.clusters_nodes", "feature"))
+ Feature(run=load("s3.tests.export_part.engines_volumes", "feature"))
+ Feature(run=load("s3.tests.export_part.datatypes", "feature"))
+ Feature(run=load("s3.tests.export_part.concurrency_networks", "feature"))
+ Feature(run=load("s3.tests.export_part.system_monitoring", "feature"))
diff --git a/s3/tests/export_part/sanity.py b/s3/tests/export_part/sanity.py
new file mode 100644
index 000000000..22809de2e
--- /dev/null
+++ b/s3/tests/export_part/sanity.py
@@ -0,0 +1,312 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.queries import *
+from s3.requirements.export_part import *
+from alter.table.replace_partition.partition_types import (
+ table_with_compact_and_wide_parts,
+)
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Settings_AllowExperimental("1.0"))
+def export_setting(self):
+ """Check that the export setting is settable in 2 ways when exporting parts."""
+
+ with Given("I create a populated source table and 2 empty S3 tables"):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name1 = create_s3_table(table_name="s3_1", create_new_bucket=True)
+ s3_table_name2 = create_s3_table(table_name="s3_2")
+
+ with When("I export parts to the first S3 table using the SET query"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name1,
+ node=self.context.node,
+ inline_settings=True,
+ )
+
+ with And("I export parts to the second S3 table using the settings argument"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name2,
+ node=self.context.node,
+ inline_settings=False,
+ settings=self.context.default_settings,
+ )
+
+ with And("I read data from all tables"):
+ source_data = select_all_ordered(
+ table_name=source_table, node=self.context.node
+ )
+ destination_data1 = select_all_ordered(
+ table_name=s3_table_name1, node=self.context.node
+ )
+ destination_data2 = select_all_ordered(
+ table_name=s3_table_name2, node=self.context.node
+ )
+
+ with Then("All tables should have the same data"):
+ assert source_data == destination_data1, error()
+ assert source_data == destination_data2, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_SchemaCompatibility("1.0"))
+def mismatched_columns(self):
+ """Test exporting parts when source and destination tables have mismatched columns."""
+
+ with Given("I create a source table and S3 table with different columns"):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=default_columns(simple=False),
+ )
+
+ with When("I export parts to the S3 table"):
+ results = export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to mismatched columns"):
+ assert results[0].exitcode == 122, error()
+ assert "Tables have different structure" in results[0].output, error()
+
+
+@TestScenario
+@Requirements(
+ RQ_ClickHouse_ExportPart_SQLCommand("1.0"),
+)
+def basic_table(self):
+ """Test exporting parts of a basic table."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_EmptyTable("1.0"))
+def empty_table(self):
+ """Test exporting parts from an empty table."""
+
+ with Given("I create empty source and S3 tables"):
+ partitioned_merge_tree_table(
+ table_name="empty_source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=False,
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="empty_s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="empty_source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read data from both tables"):
+ source_data = select_all_ordered(
+ table_name="empty_source", node=self.context.node
+ )
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+
+ with Then("They should be empty"):
+ assert source_data == [], error()
+ assert destination_data == [], error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_PartitionKeyTypes("1.0"))
+def no_partition_by(self):
+ """Test exporting parts when the source table has no PARTITION BY type."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="tuple()",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3", create_new_bucket=True, partition_by="tuple()"
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_PartTypes("1.0"))
+def wide_and_compact_parts(self):
+ """Check that exporting with both wide and compact parts is supported."""
+
+ with Given("I create a source table with wide and compact parts"):
+ source_table = "source_" + getuid()
+
+ table_with_compact_and_wide_parts(table_name=source_table)
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_SchemaChangeIsolation("1.0"))
+def export_and_drop(self):
+ """Check that dropping a column immediately after export doesn't affect exported data."""
+
+ with Given(
+ "I create a populated source table and empty S3 table",
+ description="""
+ Stop merges must be false to allow for mutations like dropping a column.
+ """,
+ ):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=False,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export data"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read the source before dropping a column"):
+ source_data = select_all_ordered(
+ table_name=source_table, node=self.context.node
+ )
+
+ with And("I drop a source column"):
+ drop_column(
+ node=self.context.node,
+ table_name=source_table,
+ column_name="i",
+ )
+
+ with Then("Check source before drop matches destination"):
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+ assert source_data == destination_data, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_LargeParts("1.0"))
+def large_part(self):
+ """Test exporting a large part."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ source_table = "source_" + getuid()
+
+ partitioned_merge_tree_table(
+ table_name=source_table,
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ number_of_values=100000000,
+ number_of_parts=1,
+ number_of_partitions=1,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ )
+
+
+@TestFeature
+@Name("sanity")
+def feature(self):
+ """Check basic functionality of exporting data parts to S3 storage."""
+
+ Scenario(run=empty_table)
+ Scenario(run=basic_table)
+ Scenario(run=no_partition_by)
+ Scenario(run=mismatched_columns)
+ Scenario(run=wide_and_compact_parts)
+ Scenario(run=export_and_drop)
+ if self.context.stress:
+ Scenario(run=large_part)
+ # Scenario(run=export_setting) # This test fails because of an actual bug in the export setting
diff --git a/s3/tests/export_part/steps.py b/s3/tests/export_part/steps.py
new file mode 100644
index 000000000..0bea888c8
--- /dev/null
+++ b/s3/tests/export_part/steps.py
@@ -0,0 +1,368 @@
+import json
+
+from testflows.core import *
+from testflows.asserts import error
+from helpers.common import getuid
+from helpers.create import *
+from helpers.queries import *
+from s3.tests.common import temporary_bucket_path, s3_storage
+
+
+@TestStep(Given)
+def minio_storage_configuration(self, restart=True):
+ """Create storage configuration with jbod disks, MinIO S3 disk, and tiered storage policy."""
+ with Given(
+ "I configure storage with jbod disks, MinIO S3 disk, and tiered storage"
+ ):
+ disks = {
+ "jbod1": {"path": "/jbod1/"},
+ "jbod2": {"path": "/jbod2/"},
+ "jbod3": {"path": "/jbod3/"},
+ "jbod4": {"path": "/jbod4/"},
+ "external": {"path": "/external/"},
+ "external2": {"path": "/external2/"},
+ "minio": {
+ "type": "s3",
+ "endpoint": "http://minio1:9001/root/data/",
+ "access_key_id": "minio_user",
+ "secret_access_key": "minio123",
+ },
+ "s3_cache": {
+ "type": "cache",
+ "disk": "minio",
+ "path": "minio_cache/",
+ "max_size": "22548578304",
+ "cache_on_write_operations": "1",
+ },
+ }
+
+ policies = {
+ "jbod1": {"volumes": {"main": {"disk": "jbod1"}}},
+ "jbod2": {"volumes": {"main": {"disk": "jbod2"}}},
+ "jbod3": {"volumes": {"main": {"disk": "jbod3"}}},
+ "jbod4": {"volumes": {"main": {"disk": "jbod4"}}},
+ "external": {"volumes": {"main": {"disk": "external"}}},
+ "external2": {"volumes": {"main": {"disk": "external2"}}},
+ "tiered_storage": {
+ "volumes": {
+ "hot": [
+ {"disk": "jbod1"},
+ {"disk": "jbod2"},
+ {"max_data_part_size_bytes": "2048"},
+ ],
+ "cold": [
+ {"disk": "external"},
+ {"disk": "external2"},
+ ],
+ },
+ "move_factor": "0.7",
+ },
+ "s3_cache": {"volumes": {"external": {"disk": "s3_cache"}}},
+ "minio_external_nocache": {"volumes": {"external": {"disk": "minio"}}},
+ }
+
+ s3_storage(disks=disks, policies=policies, restart=restart)
+
+
+def default_columns(simple=True, partition_key_type="UInt8"):
+ columns = [
+ {"name": "p", "type": partition_key_type},
+ {"name": "i", "type": "UInt64"},
+ {"name": "Path", "type": "String"},
+ {"name": "Time", "type": "DateTime"},
+ {"name": "Value", "type": "Float64"},
+ {"name": "Timestamp", "type": "Int64"},
+ ]
+
+ if simple:
+ return columns[:2]
+ else:
+ return columns
+
+
+def valid_partition_key_types_columns():
+ return [
+ {"name": "int8", "type": "Int8"},
+ {"name": "int16", "type": "Int16"},
+ {"name": "int32", "type": "Int32"},
+ {"name": "int64", "type": "Int64"},
+ {"name": "uint8", "type": "UInt8"},
+ {"name": "uint16", "type": "UInt16"},
+ {"name": "uint32", "type": "UInt32"},
+ {"name": "uint64", "type": "UInt64"},
+ {"name": "date", "type": "Date"},
+ {"name": "date32", "type": "Date32"},
+ {"name": "datetime", "type": "DateTime"},
+ {"name": "datetime64", "type": "DateTime64"},
+ {"name": "string", "type": "String"},
+ {"name": "fixedstring", "type": "FixedString(10)"},
+ ]
+
+
+@TestStep(Given)
+def create_temp_bucket(self):
+ """Create temporary S3 bucket."""
+
+ temp_s3_path = temporary_bucket_path(
+ bucket_prefix=f"{self.context.bucket_prefix}/export_part"
+ )
+
+ self.context.uri = f"{self.context.uri_base}export_part/{temp_s3_path}/"
+
+
+@TestStep(Given)
+def create_s3_table(
+ self,
+ table_name,
+ cluster=None,
+ create_new_bucket=False,
+ columns=None,
+ partition_by="p",
+):
+ """Create a destination S3 table."""
+
+ if create_new_bucket:
+ create_temp_bucket()
+
+ if columns is None:
+ columns = default_columns(simple=True)
+
+ table_name = f"{table_name}_{getuid()}"
+ engine = f"""
+ S3(
+ '{self.context.uri}',
+ '{self.context.access_key_id}',
+ '{self.context.secret_access_key}',
+ filename='{table_name}',
+ format='Parquet',
+ compression='auto',
+ partition_strategy='hive'
+ )
+ """
+
+ create_table(
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ engine=engine,
+ cluster=cluster,
+ )
+
+ return table_name
+
+
+@TestStep(When)
+def kill_minio(self, cluster=None, container_name="s3_env-minio1-1", signal="KILL"):
+ """Forcefully kill MinIO container to simulate network crash."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ retry(cluster.command, 5)(
+ None,
+ f"docker kill --signal={signal} {container_name}",
+ timeout=60,
+ exitcode=0,
+ steps=False,
+ )
+
+ if signal == "TERM":
+ with And("Waiting for MinIO container to stop"):
+ for attempt in retries(timeout=30, delay=1):
+ with attempt:
+ result = cluster.command(
+ None,
+ f"docker ps --filter name={container_name} --format '{{{{.Names}}}}'",
+ timeout=10,
+ steps=False,
+ no_checks=True,
+ )
+ if container_name not in result.output:
+ break
+ fail("MinIO container still running")
+
+
+@TestStep(When)
+def start_minio(self, cluster=None, container_name="s3_env-minio1-1"):
+ """Start MinIO container and wait for it to be ready."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ with By("Starting MinIO container"):
+ retry(cluster.command, 5)(
+ None,
+ f"docker start {container_name}",
+ timeout=60,
+ exitcode=0,
+ steps=True,
+ )
+
+ with And("Waiting for MinIO to be ready"):
+ for attempt in retries(timeout=30, delay=1):
+ with attempt:
+ result = cluster.command(
+ None,
+ f"docker exec {container_name} curl -f http://localhost:9001/minio/health/live",
+ timeout=10,
+ steps=False,
+ no_checks=True,
+ )
+ if result.exitcode != 0:
+ fail("MinIO health check failed")
+
+
+@TestStep(When)
+def get_parts(self, table_name, node):
+ """Get all parts for a table on a given node."""
+
+ output = node.query(
+ f"SELECT name FROM system.parts WHERE table = '{table_name}'",
+ exitcode=0,
+ steps=True,
+ ).output
+ return sorted([row.strip() for row in output.splitlines()])
+
+
+@TestStep(When)
+def export_parts(
+ self,
+ source_table,
+ destination_table,
+ node,
+ parts=None,
+ exitcode=0,
+ settings=None,
+ inline_settings=True,
+):
+ """Export parts from a source table to a destination table on the same node. If parts are not provided, all parts will be exported."""
+
+ if parts is None:
+ parts = get_parts(table_name=source_table, node=node)
+
+ if inline_settings is True:
+ inline_settings = self.context.default_settings
+
+ no_checks = exitcode != 0
+ output = []
+
+ for part in parts:
+ output.append(
+ node.query(
+ f"ALTER TABLE {source_table} EXPORT PART '{part}' TO TABLE {destination_table}",
+ exitcode=exitcode,
+ no_checks=no_checks,
+ steps=True,
+ settings=settings,
+ inline_settings=inline_settings,
+ )
+ )
+
+ return output
+
+
+@TestStep(When)
+def get_export_events(self, node):
+ """Get the export data from the system.events table of a given node."""
+
+ output = node.query(
+ "SELECT name, value FROM system.events WHERE name LIKE '%%Export%%' FORMAT JSONEachRow",
+ exitcode=0,
+ steps=True,
+ ).output
+
+ events = {}
+ for line in output.strip().splitlines():
+ event = json.loads(line)
+ events[event["name"]] = int(event["value"])
+
+ if "PartsExportFailures" not in events:
+ events["PartsExportFailures"] = 0
+ if "PartsExports" not in events:
+ events["PartsExports"] = 0
+ if "PartsExportDuplicated" not in events:
+ events["PartsExportDuplicated"] = 0
+
+ return events
+
+
+@TestStep(When)
+def get_part_log(self, node):
+ """Get the part log from the system.part_log table of a given node."""
+
+ output = node.query(
+ "SELECT part_name FROM system.part_log WHERE event_type = 'ExportPart'",
+ exitcode=0,
+ steps=True,
+ ).output.splitlines()
+
+ return output
+
+
+@TestStep(When)
+def get_system_exports(self, node):
+ """Get the system.exports source and destination table columns for all ongoing exports."""
+
+ exports = node.query(
+ "SELECT source_table, destination_table FROM system.exports",
+ exitcode=0,
+ steps=True,
+ ).output.splitlines()
+
+ return [line.strip().split("\t") for line in exports]
+
+
+@TestStep(Then)
+def source_matches_destination(
+ self, source_table, destination_table, source_node=None, destination_node=None
+):
+ """Check that source and destination table data matches."""
+
+ if source_node is None:
+ source_node = self.context.node
+ if destination_node is None:
+ destination_node = self.context.node
+
+ source_data = select_all_ordered(table_name=source_table, node=source_node)
+ destination_data = select_all_ordered(
+ table_name=destination_table, node=destination_node
+ )
+ assert source_data == destination_data, error()
+
+
+@TestStep(Then)
+def verify_export_concurrency(self, node, source_tables):
+ """Verify exports from different tables ran concurrently by checking overlapping execution times.
+
+ Checks that for each table, there's at least one pair of consecutive exports from that table
+ with an export from another table in between, confirming concurrent execution.
+ """
+
+ table_filter = " OR ".join([f"table = '{table}'" for table in source_tables])
+
+ query = f"""
+ SELECT
+ table
+ FROM system.part_log
+ WHERE event_type = 'ExportPart'
+ AND ({table_filter})
+ ORDER BY event_time_microseconds
+ """
+
+ result = node.query(query, exitcode=0, steps=True)
+
+ exports = [line for line in result.output.strip().splitlines()]
+
+ tables_done = set()
+
+ for i in range(len(exports) - 1):
+ current_table = exports[i]
+ next_table = exports[i + 1]
+
+ if current_table != next_table and current_table not in tables_done:
+ for j in range(i + 2, len(exports)):
+ if exports[j] == current_table:
+ tables_done.add(current_table)
+ break
+
+ assert len(tables_done) == len(source_tables), error()
diff --git a/s3/tests/export_part/system_monitoring.py b/s3/tests/export_part/system_monitoring.py
new file mode 100644
index 000000000..cfbd555bf
--- /dev/null
+++ b/s3/tests/export_part/system_monitoring.py
@@ -0,0 +1,145 @@
+from time import sleep
+
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from s3.requirements.export_part import *
+from alter.stress.tests.tc_netem import *
+
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Logging("1.0"))
+def part_logging(self):
+ """Check part exports are logged correctly in both system.events and system.part_log."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I read the initial logged export events"):
+ initial_events = get_export_events(node=self.context.node)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read the final logged export events and part log"):
+ final_events = get_export_events(node=self.context.node)
+ part_log = get_part_log(node=self.context.node)
+
+ with Then("I check that the number of part exports is correct"):
+ assert (
+ final_events["PartsExports"] - initial_events["PartsExports"] == 5
+ ), error()
+
+ with And("I check that the part log contains the correct parts"):
+ parts = get_parts(table_name="source", node=self.context.node)
+ for part in parts:
+ assert part in part_log, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Idempotency("1.0"))
+def duplicate_logging(self):
+ """Check duplicate exports are logged correctly in system.events."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I read the initial export events"):
+ initial_events = get_export_events(node=self.context.node)
+
+ with When("I try to export the parts twice"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+ with And("Check logs for correct number of duplicate exports"):
+ final_events = get_export_events(node=self.context.node)
+ assert (
+ final_events["PartsExportDuplicated"]
+ - initial_events["PartsExportDuplicated"]
+ == 5
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_SystemTables_Exports("1.0"))
+def system_exports_logging(self):
+ """Check that system.exports table tracks export operations before they complete."""
+
+ with Given(
+ "I create a populated source table with large enough parts and empty S3 table"
+ ):
+ source_table = partitioned_merge_tree_table(
+ table_name=f"source_{getuid()}",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ number_of_values=1000000,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I slow down the network speed"):
+ network_packet_rate_limit(node=self.context.node, rate_mbit=250)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=source_table,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("I check that system.exports contains some relevant parts"):
+ exports = get_system_exports(node=self.context.node)
+ assert len(exports) > 0, error()
+ assert [source_table, s3_table_name] in exports, error()
+
+ with And("I verify that system.exports empties after exports complete"):
+ sleep(5)
+ assert len(get_system_exports(node=self.context.node)) == 0, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_ServerSettings_BackgroundMovePoolSize("1.0"))
+def background_move_pool_size(self):
+ pass
+
+
+@TestFeature
+@Name("system monitoring")
+@Requirements(RQ_ClickHouse_ExportPart_Logging("1.0"))
+def feature(self):
+ """Check system monitoring of export events."""
+
+ Scenario(run=part_logging)
+ Scenario(run=duplicate_logging)
+ Scenario(run=system_exports_logging)
diff --git a/s3/tests/export_partition/clusters_nodes.py b/s3/tests/export_partition/clusters_nodes.py
new file mode 100644
index 000000000..b7e280e93
--- /dev/null
+++ b/s3/tests/export_partition/clusters_nodes.py
@@ -0,0 +1,77 @@
+from itertools import combinations
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.queries import *
+from alter.table.replace_partition.common import create_partitions_with_random_uint64
+
+
+@TestScenario
+def different_nodes_same_destination(self, cluster, node1, node2):
+ """Test export part from different nodes to same S3 destination in a given cluster."""
+
+ with Given("I create an empty source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ cluster=cluster,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3", create_new_bucket=True, cluster=cluster
+ )
+
+ with And("I populate the source tables on both nodes"):
+ create_partitions_with_random_uint64(table_name="source", node=node1)
+ create_partitions_with_random_uint64(table_name="source", node=node2)
+
+ with When("I export parts to the S3 table from both nodes"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=node1,
+ )
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=node2,
+ )
+
+ with And("I read data from all tables on both nodes"):
+ source_data1 = select_all_ordered(table_name="source", node=node1)
+ source_data2 = select_all_ordered(table_name="source", node=node2)
+ destination_data1 = select_all_ordered(table_name=s3_table_name, node=node1)
+ destination_data2 = select_all_ordered(table_name=s3_table_name, node=node2)
+
+ with Then(
+ "Destination data should be comprised of data from both sources, and identical on both nodes"
+ ):
+ assert set(destination_data1) == set(source_data1) | set(source_data2), error()
+ assert set(destination_data2) == set(source_data1) | set(source_data2), error()
+
+
+@TestFeature
+@Name("clusters and nodes")
+def feature(self):
+ """Check functionality of exporting data parts to S3 storage from different clusters and nodes."""
+
+ clusters = [
+ "sharded_cluster",
+ "replicated_cluster",
+ "one_shard_cluster",
+ "sharded_cluster12",
+ "one_shard_cluster12",
+ "sharded_cluster23",
+ "one_shard_cluster23",
+ ]
+
+ for cluster in clusters:
+ with Given(f"I get nodes for cluster {cluster}"):
+ node_names = get_cluster_nodes(cluster=cluster)
+
+ for node1_name, node2_name in combinations(node_names, 2):
+ node1 = self.context.cluster.node(node1_name)
+ node2 = self.context.cluster.node(node2_name)
+ different_nodes_same_destination(cluster=cluster, node1=node1, node2=node2)
diff --git a/s3/tests/export_partition/concurrency_networks.py b/s3/tests/export_partition/concurrency_networks.py
new file mode 100644
index 000000000..0f2d8936d
--- /dev/null
+++ b/s3/tests/export_partition/concurrency_networks.py
@@ -0,0 +1,458 @@
+from testflows.core import *
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.queries import *
+from s3.requirements.export_part import *
+from alter.stress.tests.tc_netem import *
+
+
+@TestScenario
+def basic_concurrent_export(self, threads):
+ """Check concurrent exports from different sources to the same S3 table."""
+
+ with Given(f"I create {threads} populated source tables and an empty S3 table"):
+ for i in range(threads):
+ partitioned_merge_tree_table(
+ table_name=f"source{i}",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts from all sources concurrently to the S3 table"):
+ for i in range(threads):
+ Step(test=export_parts, parallel=True)(
+ source_table=f"source{i}",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ join()
+
+ with And("I read data from all tables"):
+ source_data = []
+ for i in range(threads):
+ data = select_all_ordered(table_name=f"source{i}", node=self.context.node)
+ source_data.extend(data)
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+
+ with Then("All data should be present in the S3 table"):
+ assert set(source_data) == set(destination_data), error()
+
+
+@TestScenario
+def packet_delay(self, delay_ms):
+ """Check that exports work correctly with packet delay."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply a packet delay"):
+ network_packet_delay(node=self.context.node, delay_ms=delay_ms)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_loss(self, percent_loss):
+ """Check that exports work correctly with packet loss."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet loss"):
+ network_packet_loss(node=self.context.node, percent_loss=percent_loss)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_loss_gemodel(self, interruption_probability, recovery_probability):
+ """Check that exports work correctly with packet loss using the GE model."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet loss using the GE model"):
+ network_packet_loss_gemodel(
+ node=self.context.node,
+ interruption_probability=interruption_probability,
+ recovery_probability=recovery_probability,
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_corruption(self, percent_corrupt):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet corruption"):
+ network_packet_corruption(
+ node=self.context.node, percent_corrupt=percent_corrupt
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_duplication(self, percent_duplicated):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet duplication"):
+ network_packet_duplication(
+ node=self.context.node, percent_duplicated=percent_duplicated
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_reordering(self, delay_ms, percent_reordered):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet reordering"):
+ network_packet_reordering(
+ node=self.context.node,
+ delay_ms=delay_ms,
+ percent_reordered=percent_reordered,
+ )
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def packet_rate_limit(self, rate_mbit):
+ """Check that exports work correctly with packet corruption."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I apply packet rate limit"):
+ network_packet_rate_limit(node=self.context.node, rate_mbit=rate_mbit)
+
+ with And("I export parts from the source table to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ for retry in retries(timeout=30, delay=1):
+ with retry:
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def concurrent_insert(self):
+ """Check that exports work correctly with concurrent inserts of source data."""
+
+ with Given("I create an empty source and S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When(
+ "I insert data and export it in parallel",
+ description="""
+ 5 partitions with 1 part each are inserted.
+ The export is queued in parallel and usually behaves by exporting
+ a snapshot of the source data, often getting just the first partition
+ which means the export happens right after the first INSERT query completes.
+ """,
+ ):
+ Step(test=create_partitions_with_random_uint64, parallel=True)(
+ table_name="source",
+ number_of_partitions=5,
+ number_of_parts=1,
+ )
+ Step(test=export_parts, parallel=True)(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ join()
+
+ with Then("Destination data should be a subset of source data"):
+ source_data = select_all_ordered(table_name="source", node=self.context.node)
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+ assert set(source_data) >= set(destination_data), error()
+
+ with And("Inserts should have completed successfully"):
+ assert len(source_data) == 15, error()
+
+
+@TestScenario
+def export_and_drop(self):
+ """Check that dropping a column immediately after export works correctly."""
+ pause()
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ # s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+ # drop_column(
+ # node=self.context.node,
+ # table_name="source",
+ # column_name="i",
+ # )
+
+ # with When("I export data then drop a column"):
+ # export_parts(
+ # source_table="source",
+ # destination_table=s3_table_name,
+ # node=self.context.node,
+ # )
+ # drop_column(
+ # node=self.context.node,
+ # table_name="source",
+ # column_name="i",
+ # )
+ # This drop freezes the test ☠️☠️☠️
+
+
+@TestStep(When)
+def kill_minio(self, cluster=None, container_name="s3_env-minio1-1", signal="KILL"):
+ """Forcefully kill MinIO container to simulate network crash."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ retry(cluster.command, 5)(
+ None,
+ f"docker kill --signal={signal} {container_name}",
+ timeout=60,
+ exitcode=0,
+ steps=False,
+ )
+
+
+@TestStep(When)
+def start_minio(self, cluster=None, container_name="s3_env-minio1-1", timeout=300):
+ """Start MinIO container and wait for it to be ready."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ with By("starting MinIO container"):
+ retry(cluster.command, 5)(
+ None,
+ f"docker start {container_name}",
+ timeout=timeout,
+ exitcode=0,
+ steps=True,
+ )
+
+ with And("waiting for MinIO to be ready"):
+ for attempt in retries(timeout=timeout, delay=1):
+ with attempt:
+ result = cluster.command(
+ None,
+ f"docker exec {container_name} curl -f http://localhost:9001/minio/health/live",
+ timeout=10,
+ steps=False,
+ no_checks=True,
+ )
+ if result.exitcode != 0:
+ fail("MinIO health check failed")
+
+
+@TestScenario
+def restart_minio(self):
+ """Check that restarting MinIO after exporting data works correctly."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I kill MinIO"):
+ kill_minio()
+
+ with When("I export data"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I restart MinIO"):
+ start_minio()
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestFeature
+@Requirements(RQ_ClickHouse_ExportPart_Concurrency("1.0"))
+@Name("concurrency and networks")
+def feature(self):
+ """Check that exports work correctly with concurrency and various network conditions."""
+
+ # TODO corruption (bit flipping)
+
+ Scenario(test=basic_concurrent_export)(threads=5)
+ Scenario(test=packet_delay)(delay_ms=100)
+ Scenario(test=packet_loss)(percent_loss=50)
+ Scenario(test=packet_loss_gemodel)(
+ interruption_probability=40, recovery_probability=70
+ )
+ Scenario(test=packet_corruption)(percent_corrupt=50)
+ Scenario(test=packet_duplication)(percent_duplicated=50)
+ Scenario(test=packet_reordering)(delay_ms=100, percent_reordered=90)
+ Scenario(test=packet_rate_limit)(rate_mbit=0.05)
+ Scenario(run=concurrent_insert)
+ Scenario(run=restart_minio)
+
+ # Scenario(run=export_and_drop)
diff --git a/s3/tests/export_partition/datatypes.py b/s3/tests/export_partition/datatypes.py
new file mode 100644
index 000000000..f1ee4584b
--- /dev/null
+++ b/s3/tests/export_partition/datatypes.py
@@ -0,0 +1,123 @@
+from testflows.core import *
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.queries import *
+from helpers.common import getuid
+from s3.requirements.export_part import *
+
+
+@TestStep(When)
+def insert_all_datatypes(self, table_name, rows_per_part=1, num_parts=1, node=None):
+ """Insert all datatypes into a MergeTree table."""
+
+ if node is None:
+ node = self.context.node
+
+ for part in range(num_parts):
+ node.query(
+ f"INSERT INTO {table_name} (int8, int16, int32, int64, uint8, uint16, uint32, uint64, date, date32, datetime, datetime64, string, fixedstring) SELECT 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, '13', '14' FROM numbers({rows_per_part})"
+ )
+
+
+@TestStep(Given)
+def create_merge_tree_all_valid_partition_key_types(
+ self, column_name, cluster=None, node=None, rows_per_part=1
+):
+ """Create a MergeTree table with all valid partition key types and both wide and compact parts."""
+
+ if node is None:
+ node = self.context.node
+
+ with By("creating a MergeTree table with all data types"):
+ table_name = f"table_{getuid()}"
+ create_merge_tree_table(
+ table_name=table_name,
+ columns=valid_partition_key_types_columns(),
+ partition_by=column_name,
+ cluster=cluster,
+ stop_merges=True,
+ query_settings=f"min_rows_for_wide_part=10",
+ )
+
+ with And("I insert compact and wide parts into the table"):
+ insert_all_datatypes(
+ table_name=table_name,
+ rows_per_part=rows_per_part,
+ num_parts=self.context.num_parts,
+ node=node,
+ )
+
+ return table_name
+
+
+@TestCheck
+def valid_partition_key_table(self, partition_key_type, rows_per_part=1):
+ """Check exporting to a source table with specified valid partition key type and rows."""
+
+ with Given(
+ f"I create a source table with valid partition key type {partition_key_type} and empty S3 table"
+ ):
+ table_name = create_merge_tree_all_valid_partition_key_types(
+ column_name=partition_key_type,
+ rows_per_part=rows_per_part,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=valid_partition_key_types_columns(),
+ partition_by=partition_key_type,
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table=table_name,
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read data from both tables"):
+ source_data = select_all_ordered(
+ table_name=table_name, node=self.context.node, order_by=partition_key_type
+ )
+ destination_data = select_all_ordered(
+ table_name=s3_table_name,
+ node=self.context.node,
+ order_by=partition_key_type,
+ )
+
+ with Then("They should be the same"):
+ assert source_data == destination_data, error()
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+@Requirements(RQ_ClickHouse_ExportPart_PartitionKeyTypes("1.0"))
+def valid_partition_key_types_compact(self):
+ """Check that all partition key data types are supported when exporting compact parts."""
+
+ key_types = [datatype["name"] for datatype in valid_partition_key_types_columns()]
+ valid_partition_key_table(partition_key_type=either(*key_types), rows_per_part=1)
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+def valid_partition_key_types_wide(self):
+ """Check that all partition key data types are supported when exporting wide parts."""
+
+ key_types = [datatype["name"] for datatype in valid_partition_key_types_columns()]
+ valid_partition_key_table(partition_key_type=either(*key_types), rows_per_part=100)
+
+
+@TestFeature
+@Name("datatypes")
+@Requirements(
+ RQ_ClickHouse_ExportPart_PartitionKeyTypes("1.0"),
+ RQ_ClickHouse_ExportPart_PartTypes("1.0"),
+)
+def feature(self, num_parts=10):
+ """Check that all data types are supported when exporting parts."""
+
+ self.context.num_parts = num_parts
+
+ Scenario(run=valid_partition_key_types_compact)
+ Scenario(run=valid_partition_key_types_wide)
diff --git a/s3/tests/export_partition/engines_volumes.py b/s3/tests/export_partition/engines_volumes.py
new file mode 100644
index 000000000..04e298c7b
--- /dev/null
+++ b/s3/tests/export_partition/engines_volumes.py
@@ -0,0 +1,142 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from s3.requirements.export_part import *
+from helpers.queries import *
+
+
+# TODO replicated merge tree tables (all types)
+
+
+@TestCheck
+def configured_table(self, table_engine, number_of_partitions, number_of_parts):
+ """Test a specific combination of table engine, number of partitions, and number of parts."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ table_engine(
+ table_name="source",
+ partition_by="p",
+ stop_merges=True,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ columns=default_columns(simple=False, partition_key_type="Int8"),
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=default_columns(simple=False, partition_key_type="Int8"),
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Source and destination tables should match"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+@Requirements(RQ_ClickHouse_ExportPart_SourceEngines("1.0"))
+def table_combos(self):
+ """Test various combinations of table engines, number of partitions, and number of parts."""
+
+ tables = [
+ partitioned_merge_tree_table,
+ partitioned_replacing_merge_tree_table,
+ partitioned_summing_merge_tree_table,
+ partitioned_collapsing_merge_tree_table,
+ partitioned_versioned_collapsing_merge_tree_table,
+ partitioned_aggregating_merge_tree_table,
+ partitioned_graphite_merge_tree_table,
+ ]
+ number_of_partitions = [5] if not self.context.stress else [1, 5, 10]
+ number_of_parts = [1] if not self.context.stress else [1, 5, 10]
+
+ table_engine = either(*tables)
+ number_of_partitions = either(*number_of_partitions)
+ number_of_parts = either(*number_of_parts)
+
+ Combination(
+ name=f"{table_engine.__name__} partitions={number_of_partitions} parts={number_of_parts}",
+ test=configured_table,
+ )(
+ table_engine=table_engine,
+ number_of_partitions=number_of_partitions,
+ number_of_parts=number_of_parts,
+ )
+
+
+@TestCheck
+def configured_volume(self, volume):
+ """Test a specific combination of volume."""
+
+ with Given(f"I create an empty source table on volume {volume} and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ query_settings=f"storage_policy = '{volume}'",
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I populate the source table with parts exceeding 2KB each"):
+ create_partitions_with_random_uint64(
+ table_name="source",
+ node=self.context.node,
+ number_of_values=500,
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Source and destination tables should match"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestSketch(Scenario)
+@Flags(TE)
+def volume_combos(self):
+ """Test exporting to various storage policies."""
+
+ volumes = [
+ "jbod1",
+ "jbod2",
+ "jbod3",
+ "jbod4",
+ "external",
+ "external2",
+ "tiered_storage",
+ ]
+ volume = either(*volumes)
+
+ Combination(
+ name=f"volume={volume}",
+ test=configured_volume,
+ )(
+ volume=volume,
+ )
+
+
+@TestFeature
+@Name("engines and volumes")
+def feature(self):
+ """Check exporting parts to S3 storage with different table engines and volumes."""
+
+ Scenario(run=table_combos)
+ Scenario(run=volume_combos)
diff --git a/s3/tests/export_partition/error_handling.py b/s3/tests/export_partition/error_handling.py
new file mode 100644
index 000000000..e40221bac
--- /dev/null
+++ b/s3/tests/export_partition/error_handling.py
@@ -0,0 +1,170 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.queries import *
+from s3.requirements.export_part import *
+
+
+@TestScenario
+def invalid_part_name(self):
+ """Check that exporting a non-existent part returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I create an invalid part name"):
+ invalid_part_name = "in_va_lid_part"
+
+ with When("I try to export the invalid part"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ parts=[invalid_part_name],
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to the invalid part name"):
+ assert results[0].exitcode == 233, error()
+ assert (
+ f"Unexpected part name: {invalid_part_name}" in results[0].output
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Restrictions_SameTable("1.0"))
+def same_table(self):
+ """Check exporting parts where source and destination tables are the same."""
+
+ with Given("I create a populated source table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+
+ with When("I try to export parts to itself"):
+ results = export_parts(
+ source_table="source",
+ destination_table="source",
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to same table exports"):
+ assert results[0].exitcode == 36, error()
+ assert (
+ "Exporting to the same table is not allowed" in results[0].output
+ ), error()
+
+
+@TestScenario
+def local_table(self):
+ """Test exporting parts to a local table."""
+
+ with Given("I create a populated source table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+
+ with And("I create an empty local table"):
+ partitioned_merge_tree_table(
+ table_name="destination",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ populate=False,
+ )
+
+ with When("I export parts to the local table"):
+ results = export_parts(
+ source_table="source",
+ destination_table="destination",
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to local table exports"):
+ assert results[0].exitcode == 48, error()
+ assert (
+ "Destination storage MergeTree does not support MergeTree parts or uses unsupported partitioning"
+ in results[0].output
+ ), error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Settings_AllowExperimental("1.0"))
+def disable_export_setting(self):
+ """Check that exporting parts without the export setting set returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I try to export the parts with the export setting disabled"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ explicit_set=-1,
+ )
+
+ with Then("I should see an error related to the export setting"):
+ assert results[0].exitcode == 88, error()
+ assert "Exporting merge tree part is experimental" in results[0].output, error()
+
+
+@TestScenario
+def different_partition_key(self):
+ """Check exporting parts with a different partition key returns the correct error."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="i",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I try to export the parts"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to the different partition key"):
+ assert results[0].exitcode == 36, error()
+ assert "Tables have different partition key" in results[0].output, error()
+
+
+@TestFeature
+@Name("error handling")
+@Requirements(RQ_ClickHouse_ExportPart_FailureHandling("1.0"))
+def feature(self):
+ """Check correct error handling when exporting parts."""
+
+ Scenario(run=invalid_part_name)
+ Scenario(run=same_table)
+ Scenario(run=local_table)
+ Scenario(run=disable_export_setting)
+ Scenario(run=different_partition_key)
diff --git a/s3/tests/export_partition/feature.py b/s3/tests/export_partition/feature.py
new file mode 100644
index 000000000..60053f5f8
--- /dev/null
+++ b/s3/tests/export_partition/feature.py
@@ -0,0 +1,22 @@
+from testflows.core import *
+from s3.requirements.export_partition import *
+
+
+@TestFeature
+@Specifications(SRS_016_ClickHouse_Export_Partition_to_S3)
+@Requirements()
+@Name("export partition")
+def minio(self, uri, bucket_prefix):
+ """Export partition suite."""
+
+ self.context.uri_base = uri
+ self.context.bucket_prefix = bucket_prefix
+ self.context.default_settings = [("allow_experimental_export_merge_tree_part", 1)]
+
+ Feature(run=load("s3.tests.export_partition.sanity", "feature"))
+ Feature(run=load("s3.tests.export_partition.error_handling", "feature"))
+ Feature(run=load("s3.tests.export_partition.clusters_nodes", "feature"))
+ Feature(run=load("s3.tests.export_partition.engines_volumes", "feature"))
+ Feature(run=load("s3.tests.export_partition.datatypes", "feature"))
+ Feature(run=load("s3.tests.export_partition.concurrency_networks", "feature"))
+ Feature(run=load("s3.tests.export_partition.system_monitoring", "feature"))
diff --git a/s3/tests/export_partition/sanity.py b/s3/tests/export_partition/sanity.py
new file mode 100644
index 000000000..3463bdd8d
--- /dev/null
+++ b/s3/tests/export_partition/sanity.py
@@ -0,0 +1,244 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from helpers.create import *
+from helpers.common import getuid
+from helpers.queries import *
+from s3.requirements.export_part import *
+from alter.table.replace_partition.partition_types import (
+ table_with_compact_and_wide_parts,
+)
+from s3.tests.export_partition.steps import export_partitions
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_Settings_AllowExperimental("1.0"))
+def export_setting(self):
+ """Check that the export setting is settable in 2 ways when exporting parts."""
+
+ with Given("I create a populated source table and 2 empty S3 tables"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name1 = create_s3_table(table_name="s3_1", create_new_bucket=True)
+ s3_table_name2 = create_s3_table(table_name="s3_2")
+
+ with When("I export parts to the first S3 table using the SET query"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name1,
+ node=self.context.node,
+ explicit_set=1,
+ )
+
+ with And("I export parts to the second S3 table using the settings argument"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name2,
+ node=self.context.node,
+ explicit_set=0,
+ )
+
+ with And("I read data from all tables"):
+ source_data = select_all_ordered(table_name="source", node=self.context.node)
+ destination_data1 = select_all_ordered(
+ table_name=s3_table_name1, node=self.context.node
+ )
+ destination_data2 = select_all_ordered(
+ table_name=s3_table_name2, node=self.context.node
+ )
+
+ with Then("All tables should have the same data"):
+ assert source_data == destination_data1, error()
+ assert source_data == destination_data2, error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_SchemaCompatibility("1.0"))
+def mismatched_columns(self):
+ """Test exporting parts when source and destination tables have mismatched columns."""
+
+ with Given("I create a source table and S3 table with different columns"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3",
+ create_new_bucket=True,
+ columns=default_columns(simple=False),
+ )
+
+ with When("I export parts to the S3 table"):
+ results = export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ exitcode=1,
+ )
+
+ with Then("I should see an error related to mismatched columns"):
+ assert results[0].exitcode == 122, error()
+ assert "Tables have different structure" in results[0].output, error()
+
+
+@TestScenario
+@Requirements()
+def basic_table(self):
+ """Test exporting partitions of a basic table."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ source_table = partitioned_replicated_merge_tree_table(
+ table_name=f"source_{getuid()}",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export partitions to the S3 table"):
+ export_partitions(
+ source_table=source_table,
+ destination_table=s3_table,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table=source_table,
+ destination_table=s3_table,
+ )
+
+
+@TestScenario
+def empty_table(self):
+ """Test exporting parts from an empty table."""
+
+ with Given("I create empty source and S3 tables"):
+ partitioned_merge_tree_table(
+ table_name="empty_source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=False,
+ populate=False,
+ )
+ s3_table_name = create_s3_table(table_name="empty_s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="empty_source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with And("I read data from both tables"):
+ source_data = select_all_ordered(
+ table_name="empty_source", node=self.context.node
+ )
+ destination_data = select_all_ordered(
+ table_name=s3_table_name, node=self.context.node
+ )
+
+ with Then("They should be empty"):
+ assert source_data == [], error()
+ assert destination_data == [], error()
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_PartitionKeyTypes("1.0"))
+def no_partition_by(self):
+ """Test exporting parts when the source table has no PARTITION BY type."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="tuple()",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(
+ table_name="s3", create_new_bucket=True, partition_by="tuple()"
+ )
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+@Requirements(RQ_ClickHouse_ExportPart_PartTypes("1.0"))
+def wide_and_compact_parts(self):
+ """Check that exporting with both wide and compact parts is supported."""
+
+ with Given("I create a source table with wide and compact parts"):
+ table_with_compact_and_wide_parts(table_name="source")
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestScenario
+def large_export(self):
+ """Test exporting a large part."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ number_of_parts=100,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I export parts to the S3 table"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+
+@TestFeature
+@Name("sanity")
+def feature(self):
+ """Check basic functionality of exporting data parts to S3 storage."""
+
+ Scenario(run=empty_table)
+ Scenario(run=basic_table)
+ Scenario(run=no_partition_by)
+ Scenario(run=mismatched_columns)
+ Scenario(run=wide_and_compact_parts)
+ if self.context.stress:
+ Scenario(run=large_export)
diff --git a/s3/tests/export_partition/steps.py b/s3/tests/export_partition/steps.py
new file mode 100644
index 000000000..f4c7c697c
--- /dev/null
+++ b/s3/tests/export_partition/steps.py
@@ -0,0 +1,369 @@
+import json
+
+from testflows.core import *
+from testflows.asserts import error
+from helpers.common import getuid
+from helpers.create import *
+from helpers.queries import *
+from s3.tests.common import temporary_bucket_path, s3_storage
+
+
+@TestStep(Given)
+def minio_storage_configuration(self, restart=True):
+ """Create storage configuration with jbod disks, MinIO S3 disk, and tiered storage policy."""
+ with Given(
+ "I configure storage with jbod disks, MinIO S3 disk, and tiered storage"
+ ):
+ disks = {
+ "jbod1": {"path": "/jbod1/"},
+ "jbod2": {"path": "/jbod2/"},
+ "jbod3": {"path": "/jbod3/"},
+ "jbod4": {"path": "/jbod4/"},
+ "external": {"path": "/external/"},
+ "external2": {"path": "/external2/"},
+ "minio": {
+ "type": "s3",
+ "endpoint": "http://minio1:9001/root/data/",
+ "access_key_id": "minio_user",
+ "secret_access_key": "minio123",
+ },
+ "s3_cache": {
+ "type": "cache",
+ "disk": "minio",
+ "path": "minio_cache/",
+ "max_size": "22548578304",
+ "cache_on_write_operations": "1",
+ },
+ }
+
+ policies = {
+ "jbod1": {"volumes": {"main": {"disk": "jbod1"}}},
+ "jbod2": {"volumes": {"main": {"disk": "jbod2"}}},
+ "jbod3": {"volumes": {"main": {"disk": "jbod3"}}},
+ "jbod4": {"volumes": {"main": {"disk": "jbod4"}}},
+ "external": {"volumes": {"main": {"disk": "external"}}},
+ "external2": {"volumes": {"main": {"disk": "external2"}}},
+ "tiered_storage": {
+ "volumes": {
+ "hot": [
+ {"disk": "jbod1"},
+ {"disk": "jbod2"},
+ {"max_data_part_size_bytes": "2048"},
+ ],
+ "cold": [
+ {"disk": "external"},
+ {"disk": "external2"},
+ ],
+ },
+ "move_factor": "0.7",
+ },
+ "s3_cache": {"volumes": {"external": {"disk": "s3_cache"}}},
+ "minio_external_nocache": {"volumes": {"external": {"disk": "minio"}}},
+ }
+
+ s3_storage(disks=disks, policies=policies, restart=restart)
+
+
+def default_columns(simple=True, partition_key_type="UInt8"):
+ columns = [
+ {"name": "p", "type": partition_key_type},
+ {"name": "i", "type": "UInt64"},
+ {"name": "Path", "type": "String"},
+ {"name": "Time", "type": "DateTime"},
+ {"name": "Value", "type": "Float64"},
+ {"name": "Timestamp", "type": "Int64"},
+ ]
+
+ if simple:
+ return columns[:2]
+ else:
+ return columns
+
+
+def valid_partition_key_types_columns():
+ return [
+ {"name": "int8", "type": "Int8"},
+ {"name": "int16", "type": "Int16"},
+ {"name": "int32", "type": "Int32"},
+ {"name": "int64", "type": "Int64"},
+ {"name": "uint8", "type": "UInt8"},
+ {"name": "uint16", "type": "UInt16"},
+ {"name": "uint32", "type": "UInt32"},
+ {"name": "uint64", "type": "UInt64"},
+ {"name": "date", "type": "Date"},
+ {"name": "date32", "type": "Date32"},
+ {"name": "datetime", "type": "DateTime"},
+ {"name": "datetime64", "type": "DateTime64"},
+ {"name": "string", "type": "String"},
+ {"name": "fixedstring", "type": "FixedString(10)"},
+ ]
+
+
+@TestStep(Given)
+def create_temp_bucket(self):
+ """Create temporary S3 bucket."""
+
+ temp_s3_path = temporary_bucket_path(
+ bucket_prefix=f"{self.context.bucket_prefix}/export_part"
+ )
+
+ self.context.uri = f"{self.context.uri_base}export_part/{temp_s3_path}/"
+
+
+@TestStep(Given)
+def create_s3_table(
+ self,
+ table_name,
+ cluster=None,
+ create_new_bucket=False,
+ columns=None,
+ partition_by="p",
+):
+ """Create a destination S3 table."""
+
+ if create_new_bucket:
+ create_temp_bucket()
+
+ if columns is None:
+ columns = default_columns(simple=True)
+
+ table_name = f"{table_name}_{getuid()}"
+ engine = f"""
+ S3(
+ '{self.context.uri}',
+ '{self.context.access_key_id}',
+ '{self.context.secret_access_key}',
+ filename='{table_name}',
+ format='Parquet',
+ compression='auto',
+ partition_strategy='hive'
+ )
+ """
+
+ create_table(
+ table_name=table_name,
+ columns=columns,
+ partition_by=partition_by,
+ engine=engine,
+ cluster=cluster,
+ )
+
+ return table_name
+
+
+@TestStep(When)
+def kill_minio(self, cluster=None, container_name="s3_env-minio1-1", signal="KILL"):
+ """Forcefully kill MinIO container to simulate network crash."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ retry(cluster.command, 5)(
+ None,
+ f"docker kill --signal={signal} {container_name}",
+ timeout=60,
+ exitcode=0,
+ steps=False,
+ )
+
+ if signal == "TERM":
+ with And("Waiting for MinIO container to stop"):
+ for attempt in retries(timeout=30, delay=1):
+ with attempt:
+ result = cluster.command(
+ None,
+ f"docker ps --filter name={container_name} --format '{{{{.Names}}}}'",
+ timeout=10,
+ steps=False,
+ no_checks=True,
+ )
+ if container_name not in result.output:
+ break
+ fail("MinIO container still running")
+
+
+@TestStep(When)
+def start_minio(self, cluster=None, container_name="s3_env-minio1-1"):
+ """Start MinIO container and wait for it to be ready."""
+
+ if cluster is None:
+ cluster = self.context.cluster
+
+ with By("Starting MinIO container"):
+ retry(cluster.command, 5)(
+ None,
+ f"docker start {container_name}",
+ timeout=60,
+ exitcode=0,
+ steps=True,
+ )
+
+ with And("Waiting for MinIO to be ready"):
+ for attempt in retries(timeout=30, delay=1):
+ with attempt:
+ result = cluster.command(
+ None,
+ f"docker exec {container_name} curl -f http://localhost:9001/minio/health/live",
+ timeout=10,
+ steps=False,
+ no_checks=True,
+ )
+ if result.exitcode != 0:
+ fail("MinIO health check failed")
+
+
+@TestStep(When)
+def get_parts(self, table_name, node):
+ """Get all parts for a table on a given node."""
+
+ output = node.query(
+ f"SELECT name FROM system.parts WHERE table = '{table_name}'",
+ exitcode=0,
+ steps=True,
+ ).output
+ return sorted([row.strip() for row in output.splitlines()])
+
+
+@TestStep(When)
+def export_partitions(
+ self,
+ source_table,
+ destination_table,
+ node,
+ parts=None,
+ exitcode=0,
+ settings=None,
+ inline_settings=True,
+):
+ """Export partitions from a source table to a destination table on the same node. If partitions are not provided, all partitions will be exported."""
+
+ if parts is None:
+ parts = get_parts(table_name=source_table, node=node)
+
+ if inline_settings is True:
+ inline_settings = self.context.default_settings
+
+ no_checks = exitcode != 0
+
+ output = []
+
+ for part in parts:
+ output.append(
+ node.query(
+ f"ALTER TABLE {source_table} EXPORT PART '{part}' TO TABLE {destination_table}",
+ exitcode=exitcode,
+ no_checks=no_checks,
+ steps=True,
+ settings=settings,
+ inline_settings=inline_settings,
+ )
+ )
+
+ return output
+
+
+@TestStep(When)
+def get_export_events(self, node):
+ """Get the export data from the system.events table of a given node."""
+
+ output = node.query(
+ "SELECT name, value FROM system.events WHERE name LIKE '%%Export%%' FORMAT JSONEachRow",
+ exitcode=0,
+ steps=True,
+ ).output
+
+ events = {}
+ for line in output.strip().splitlines():
+ event = json.loads(line)
+ events[event["name"]] = int(event["value"])
+
+ if "PartsExportFailures" not in events:
+ events["PartsExportFailurget_export_eventses"] = 0
+ if "PartsExports" not in events:
+ events["PartsExports"] = 0
+ if "PartsExportDuplicated" not in events:
+ events["PartsExportDuplicated"] = 0
+
+ return events
+
+
+@TestStep(When)
+def get_part_log(self, node):
+ """Get the part log from the system.part_log table of a given node."""
+
+ output = node.query(
+ "SELECT part_name FROM system.part_log WHERE event_type = 'ExportPart'",
+ exitcode=0,
+ steps=True,
+ ).output.splitlines()
+
+ return output
+
+
+@TestStep(When)
+def get_system_exports(self, node):
+ """Get the system.exports source and destination table columns for all ongoing exports."""
+
+ exports = node.query(
+ "SELECT source_table, destination_table FROM system.exports",
+ exitcode=0,
+ steps=True,
+ ).output.splitlines()
+
+ return [line.strip().split("\t") for line in exports]
+
+
+@TestStep(Then)
+def source_matches_destination(
+ self, source_table, destination_table, source_node=None, destination_node=None
+):
+ """Check that source and destination table data matches."""
+
+ if source_node is None:
+ source_node = self.context.node
+ if destination_node is None:
+ destination_node = self.context.node
+
+ source_data = select_all_ordered(table_name=source_table, node=source_node)
+ destination_data = select_all_ordered(
+ table_name=destination_table, node=destination_node
+ )
+ assert source_data == destination_data, error()
+
+
+@TestStep(Then)
+def verify_export_concurrency(self, node, source_tables):
+ """Verify exget_export_eventsports from different tables ran concurrently by checking overlapping execution times.
+
+ Checks that for each table, there's at least one pair of consecutive exports from that table
+ with an export from another table in between, confirming concurrent execution.
+ """
+
+ table_filter = " OR ".join([f"table = '{table}'" for table in source_tables])
+
+ query = f"""
+ SELECT
+ table
+ FROM system.part_log
+ WHERE event_type = 'ExportPart'
+ AND ({table_filter})
+ ORDER BY event_time_microseconds
+ """
+
+ result = node.query(query, exitcode=0, steps=True)
+
+ exports = [line for line in result.output.strip().splitlines()]
+
+ tables_done = set()
+
+ for i in range(len(exports) - 1):
+ current_table = exports[i]
+ next_table = exports[i + 1]
+
+ if current_table != next_table and current_table not in tables_done:
+ for j in range(i + 2, len(exports)):
+ if exports[j] == current_table:
+ tables_done.add(current_table)
+ break
+
+ assert len(tables_done) == len(source_tables), error()
diff --git a/s3/tests/export_partition/system_monitoring.py b/s3/tests/export_partition/system_monitoring.py
new file mode 100644
index 000000000..51fdce972
--- /dev/null
+++ b/s3/tests/export_partition/system_monitoring.py
@@ -0,0 +1,101 @@
+from testflows.core import *
+from testflows.asserts import error
+from s3.tests.export_part.steps import *
+from s3.requirements.export_part import *
+
+
+# TODO
+# part_log is where to look
+# overwrite file
+# max bandwidth
+# some of system.events stuff wont appear unless i set this maybe? just a guess
+# system.events
+# Export row in system.metrics??
+# partsexports incrementing correctly
+# duplicates incrementing correctly
+
+
+@TestScenario
+def part_exports(self):
+ """Check part exports are properly tracked in system.part_log."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with And("I read the initial logged number of part exports"):
+ initial_exports = get_export_events(
+ node=self.context.node
+ ) # .get("PartsExports", 0)
+ note(f"Initial exports: {initial_exports}")
+
+ # with When("I export parts to the S3 table"):
+ # export_parts(
+ # source_table="source",
+ # destination_table=s3_table_name,
+ # node=self.context.node,
+ # )
+
+ # with And("I read the final logged number of part exports"):
+ # final_exports = get_export_events(node=self.context.node).get("PartsExports", 0)
+
+ # with Then("I check that the number of part exports is correct"):
+
+ # with By("Reading the number of parts for the source table"):
+ # num_parts = len(get_parts(table_name="source", node=self.context.node))
+
+ # with And("Checking that the before and after difference is correct"):
+ # assert final_exports - initial_exports == num_parts, error()
+
+
+@TestScenario
+def duplicate_exports(self):
+ """Check duplicate exports are ignored and not exported again."""
+
+ with Given("I create a populated source table and empty S3 table"):
+ partitioned_merge_tree_table(
+ table_name="source",
+ partition_by="p",
+ columns=default_columns(),
+ stop_merges=True,
+ )
+ s3_table_name = create_s3_table(table_name="s3", create_new_bucket=True)
+
+ with When("I try to export the parts twice"):
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+ export_parts(
+ source_table="source",
+ destination_table=s3_table_name,
+ node=self.context.node,
+ )
+
+ # with And("I read the initial export events"):
+
+ with Then("Check source matches destination"):
+ source_matches_destination(
+ source_table="source",
+ destination_table=s3_table_name,
+ )
+
+ with And("Check logs for duplicate exports"):
+ export_events = get_export_events(node=self.context.node)
+ note(export_events["PartsExports"])
+
+
+@TestFeature
+@Name("system monitoring")
+@Requirements(RQ_ClickHouse_ExportPart_Logging("1.0"))
+def feature(self):
+ """Check system monitoring of export events."""
+
+ # Scenario(run=part_exports)
+ Scenario(run=duplicate_exports)