chunk-size cli args

Author: wbbradley

README.md

@@ -192,4 +192,74 @@ graph TD
     style BlobMerkle fill:#ffe1e1
     style SPM0Root fill:#fff4e1
     style Storage fill:#e1ffe1
+```
+
+### Smart Defaults - Automatic Chunk Size Selection
+
+The automatic chunk size selection behavior is based on these key parameters:
+
+1. When Chunking Kicks In
+
+Chunking is automatically used when:
+blob_size > max_blob_size_for_n_shards(n_shards, encoding_type)
+
+Where:
+- max_blob_size_for_n_shards = source_symbols_per_blob × max_symbol_size
+- max_symbol_size = 65,534 bytes (u16::MAX - 1) for RS2 encoding
+- source_symbols_per_blob = n_primary × n_secondary (depends on shard count)
+
+Example for 1000 shards:
+- Primary source symbols: 334
+- Secondary source symbols: 667
+- Total source symbols: 334 × 667 = 222,778
+- Max single-chunk size: 222,778 × 65,534 = ~13.9 GB
+
+So for a typical network with 1000 shards, chunking automatically kicks in for blobs larger than
+~13.9 GB.
+
+2. Default Chunk Size
+
+When chunking is needed, the system uses:
+pub const DEFAULT_CHUNK_SIZE: u64 = 10 * 1024 * 1024; // 10 MB
+
+This was chosen based on several factors documented in the code:
+- Memory efficiency: 10 MB chunks keep memory usage reasonable during encoding/decoding
+- Metadata overhead: At 10 MB per chunk with 1000 shards, metadata is only 0.64% overhead (64 KB
+  metadata per 10 MB chunk)
+- Streaming performance: Smaller chunks enable faster initial data delivery
+- Storage granularity: Reasonable balance between network round-trips and overhead
+
+3. Constraints
+
+The system enforces:
+- Minimum chunk size: 10 MB (prevents excessive metadata overhead)
+- Maximum chunks per blob: 1000 (bounds total metadata size to ~64 MB)
+
+4. Practical Examples
+
+Small blob (< 13.9 GB with 1000 shards):
+walrus store --epochs 5 small_file.bin  # 1 GB file
+  → Uses standard RS2 encoding (single chunk)
+  → No chunking needed
+
+Large blob (> 13.9 GB with 1000 shards):
+walrus store --epochs 5 large_file.bin  # 50 GB file
+  → Automatically uses RS2Chunked encoding
+  → Chunk size: 10 MB (DEFAULT_CHUNK_SIZE)
+  → Number of chunks: 5120 (50 GB / 10 MB)
+
+Manual override:
+walrus store --epochs 5 --chunk-size 20971520 large_file.bin  # 50 GB with 20 MB chunks
+  → Forces RS2Chunked encoding
+  → Chunk size: 20 MB (user specified)
+  → Number of chunks: 2560 (50 GB / 20 MB)
+  → Useful for systems with more memory available
+
+5. Why Manual Override is Useful
+
+- Memory-constrained environments: Use smaller chunks (e.g., 5 MB) to reduce peak memory usage
+- Performance tuning: Larger chunks (e.g., 20-50 MB) may improve throughput when memory is abundant
+- Testing: Validate chunking behavior with smaller test files by forcing chunked mode
 
+The smart defaults ensure that most users never need to think about chunking—it "just works" when
+blobs exceed single-chunk limits, while still giving advanced users control when needed.

crates/walrus-sdk/src/client.rs

@@ -919,7 +919,11 @@ impl WalrusNodeClient<SuiContractClient> {
         let walrus_store_blobs =
             WalrusStoreBlob::<String>::default_unencoded_blobs_from_slice(blobs, attributes);
         let start = Instant::now();
-        let encoded_blobs = self.encode_blobs(walrus_store_blobs, store_args.encoding_type)?;
+        let encoded_blobs = self.encode_blobs_with_chunk_size(
+            walrus_store_blobs,
+            store_args.encoding_type,
+            store_args.chunk_size,
+        )?;
         store_args.maybe_observe_encoding_latency(start.elapsed());
 
         let (failed_blobs, encoded_blobs): (Vec<_>, Vec<_>) =
@@ -965,7 +969,11 @@ impl WalrusNodeClient<SuiContractClient> {
         let walrus_store_blobs =
             WalrusStoreBlob::<String>::default_unencoded_blobs_from_slice(&blobs, &[]);
 
-        let encoded_blobs = self.encode_blobs(walrus_store_blobs, store_args.encoding_type)?;
+        let encoded_blobs = self.encode_blobs_with_chunk_size(
+            walrus_store_blobs,
+            store_args.encoding_type,
+            store_args.chunk_size,
+        )?;
         let (failed_blobs, encoded_blobs): (Vec<_>, Vec<_>) =
             encoded_blobs.into_iter().partition(|blob| blob.is_failed());
 
@@ -1011,7 +1019,11 @@ impl WalrusNodeClient<SuiContractClient> {
         let walrus_store_blobs =
             WalrusStoreBlob::<String>::default_unencoded_blobs_from_slice(blobs, &[]);
 
-        let encoded_blobs = self.encode_blobs(walrus_store_blobs, store_args.encoding_type)?;
+        let encoded_blobs = self.encode_blobs_with_chunk_size(
+            walrus_store_blobs,
+            store_args.encoding_type,
+            store_args.chunk_size,
+        )?;
         let (failed_blobs, encoded_blobs): (Vec<_>, Vec<_>) =
             encoded_blobs.into_iter().partition(|blob| blob.is_failed());
 
@@ -1041,6 +1053,17 @@ impl WalrusNodeClient<SuiContractClient> {
         &self,
         walrus_store_blobs: Vec<WalrusStoreBlob<'a, T>>,
         encoding_type: EncodingType,
+    ) -> ClientResult<Vec<WalrusStoreBlob<'a, T>>> {
+        self.encode_blobs_with_chunk_size(walrus_store_blobs, encoding_type, None)
+    }
+
+    /// Encodes multiple blobs with optional chunk size override.
+    #[tracing::instrument(skip_all, fields(count = walrus_store_blobs.len()))]
+    pub fn encode_blobs_with_chunk_size<'a, T: Debug + Clone + Send + Sync>(
+        &self,
+        walrus_store_blobs: Vec<WalrusStoreBlob<'a, T>>,
+        encoding_type: EncodingType,
+        chunk_size: Option<u64>,
     ) -> ClientResult<Vec<WalrusStoreBlob<'a, T>>> {
         if walrus_store_blobs.is_empty() {
             return Ok(Vec::new());
@@ -1075,10 +1098,11 @@ impl WalrusNodeClient<SuiContractClient> {
 
                 let multi_pb_clone = multi_pb.clone();
                 let unencoded_blob = blob.get_blob();
-                let encode_result = self.encode_pairs_and_metadata(
+                let encode_result = self.encode_pairs_and_metadata_with_chunk_size(
                     unencoded_blob,
                     encoding_type,
                     multi_pb_clone.as_ref(),
+                    chunk_size,
                 );
                 blob.with_encode_result(encode_result)
             })
@@ -1100,6 +1124,18 @@ impl WalrusNodeClient<SuiContractClient> {
         blob: &[u8],
         encoding_type: EncodingType,
         multi_pb: &MultiProgress,
+    ) -> ClientResult<(Vec<SliverPair>, VerifiedBlobMetadataWithId)> {
+        self.encode_pairs_and_metadata_with_chunk_size(blob, encoding_type, multi_pb, None)
+    }
+
+    /// Encodes a blob into sliver pairs and metadata with optional chunk size override.
+    #[tracing::instrument(skip_all)]
+    pub fn encode_pairs_and_metadata_with_chunk_size(
+        &self,
+        blob: &[u8],
+        encoding_type: EncodingType,
+        multi_pb: &MultiProgress,
+        chunk_size_override: Option<u64>,
     ) -> ClientResult<(Vec<SliverPair>, VerifiedBlobMetadataWithId)> {
         use walrus_core::encoding::{
             ChunkedBlobEncoder,
@@ -1116,10 +1152,23 @@ impl WalrusNodeClient<SuiContractClient> {
         let n_shards = self.encoding_config.n_shards();
         let max_single_chunk_size = max_blob_size_for_n_shards(n_shards, encoding_type);
 
-        let (pairs, metadata) = if blob_size > max_single_chunk_size {
-            // Use chunked encoding for large blobs
-            let (num_chunks, chunk_size) =
-                compute_chunk_parameters(blob_size, n_shards, encoding_type);
+        // If chunk_size_override is provided or blob is too large, use chunked encoding
+        let use_chunked = chunk_size_override.is_some() || blob_size > max_single_chunk_size;
+
+        let (pairs, metadata) = if use_chunked {
+            // Use chunked encoding for large blobs or when explicitly requested
+            let (num_chunks, chunk_size) = if let Some(override_size) = chunk_size_override {
+                // Validate that the override chunk size is reasonable
+                if override_size == 0 {
+                    return Err(ClientError::store_blob_internal(
+                        "chunk size must be greater than 0".to_string()
+                    ));
+                }
+                let num_chunks = (blob_size + override_size - 1) / override_size;
+                (num_chunks as u32, override_size)
+            } else {
+                compute_chunk_parameters(blob_size, n_shards, encoding_type)
+            };
 
             spinner.set_message(format!("encoding large blob ({} chunks)", num_chunks));
             tracing::info!(

crates/walrus-sdk/src/client/store_args.rs

@@ -33,6 +33,8 @@ pub struct StoreArgs {
     pub metrics: Option<Arc<ClientMetrics>>,
     /// The optional upload relay client, that allows to store the blob via the relay.
     pub upload_relay_client: Option<UploadRelayClient>,
+    /// Optional chunk size for chunked encoding, in bytes.
+    pub chunk_size: Option<u64>,
 }
 
 impl StoreArgs {
@@ -52,6 +54,7 @@ impl StoreArgs {
             post_store,
             metrics: None,
             upload_relay_client: None,
+            chunk_size: None,
         }
     }
 
@@ -69,6 +72,7 @@ impl StoreArgs {
             post_store: PostStoreAction::Keep,
             metrics: None,
             upload_relay_client: None,
+            chunk_size: None,
         }
     }
 
@@ -124,6 +128,12 @@ impl StoreArgs {
         self.metrics.as_ref()
     }
 
+    /// Sets the chunk size for chunked encoding.
+    pub fn with_chunk_size(mut self, chunk_size: Option<u64>) -> Self {
+        self.chunk_size = chunk_size;
+        self
+    }
+
     /// Convenience method for `with_store_optimizations(StoreOptimizations::none())`.
     pub fn no_store_optimizations(self) -> Self {
         self.with_store_optimizations(StoreOptimizations::none())

crates/walrus-service/src/client/cli/args.rs

@@ -1163,6 +1163,15 @@ pub struct CommonStoreOptions {
     #[arg(long, value_enum)]
     #[serde(default)]
     pub upload_mode: Option<UploadModeCli>,
+    /// The chunk size for chunked encoding, in bytes.
+    ///
+    /// When specified, forces the use of chunked encoding (RS2Chunked) for the blob, splitting it
+    /// into chunks of the specified size. This is useful for storing very large blobs that exceed
+    /// memory limits. If not specified, the SDK will automatically choose between standard and
+    /// chunked encoding based on the blob size.
+    #[arg(long)]
+    #[serde(default)]
+    pub chunk_size: Option<u64>,
 }
 
 #[serde_as]
@@ -1922,6 +1931,7 @@ mod tests {
                 upload_relay: None,
                 skip_tip_confirmation: false,
                 upload_mode: None,
+                chunk_size: None,
             },
         })
     }

crates/walrus-service/src/client/cli/runner.rs

@@ -671,6 +671,7 @@ impl ClientCommandRunner {
             upload_relay,
             confirmation,
             upload_mode,
+            chunk_size,
         }: StoreOptions,
     ) -> Result<()> {
         epoch_arg.exactly_one_is_some()?;
@@ -693,7 +694,17 @@ impl ClientCommandRunner {
             anyhow::bail!("deletable blobs cannot be shared");
         }
 
-        let encoding_type = encoding_type.unwrap_or(DEFAULT_ENCODING);
+        // If chunk_size is specified, force chunked encoding
+        let encoding_type = if chunk_size.is_some() {
+            if encoding_type.is_some() && encoding_type != Some(EncodingType::RS2Chunked) {
+                anyhow::bail!(
+                    "chunk-size can only be used with RS2Chunked encoding; remove --encoding-type or set it to rs2-chunked"
+                );
+            }
+            EncodingType::RS2Chunked
+        } else {
+            encoding_type.unwrap_or(DEFAULT_ENCODING)
+        };
 
         if dry_run {
             return Self::store_dry_run(client, files, encoding_type, epochs_ahead, self.json)
@@ -716,7 +727,8 @@ impl ClientCommandRunner {
             store_optimizations,
             persistence,
             post_store,
-        );
+        )
+        .with_chunk_size(chunk_size);
 
         if let Some(upload_relay) = upload_relay {
             let upload_relay_client = UploadRelayClient::new(
@@ -826,6 +838,7 @@ impl ClientCommandRunner {
             upload_relay,
             confirmation,
             upload_mode,
+            chunk_size,
         }: StoreOptions,
     ) -> Result<()> {
         epoch_arg.exactly_one_is_some()?;
@@ -838,10 +851,21 @@ impl ClientCommandRunner {
             anyhow::bail!("deletable blobs cannot be shared");
         }
 
-        let encoding_type = encoding_type.unwrap_or(DEFAULT_ENCODING);
         // Apply CLI upload preset to the in-memory config before building the client, if provided.
         let mut config = self.config?;
         config = apply_upload_mode_to_config(config, upload_mode.unwrap_or(UploadMode::Balanced));
+
+        // If chunk_size is specified, force chunked encoding
+        let encoding_type = if chunk_size.is_some() {
+            if encoding_type.is_some() && encoding_type != Some(EncodingType::RS2Chunked) {
+                anyhow::bail!(
+                    "chunk-size can only be used with RS2Chunked encoding; remove --encoding-type or set it to rs2-chunked"
+                );
+            }
+            EncodingType::RS2Chunked
+        } else {
+            encoding_type.unwrap_or(DEFAULT_ENCODING)
+        };
         let client = get_contract_client(config, self.wallet, self.gas_budget, &None).await?;
 
         let system_object = client.sui_client().read_client.get_system_object().await?;
@@ -872,7 +896,8 @@ impl ClientCommandRunner {
             store_optimizations,
             persistence,
             post_store,
-        );
+        )
+        .with_chunk_size(chunk_size);
 
         if let Some(upload_relay) = upload_relay {
             let upload_relay_client = UploadRelayClient::new(
@@ -1607,6 +1632,7 @@ struct StoreOptions {
     upload_relay: Option<Url>,
     confirmation: UserConfirmation,
     upload_mode: Option<UploadMode>,
+    chunk_size: Option<NonZeroUsize>,
 }
 
 impl TryFrom<CommonStoreOptions> for StoreOptions {