feat: adds RLP hex txn storage to tx pool service (#4521)

Signed-off-by: Simeon Nakov <[email protected]>

Author: simzzz

.env.http.example

@@ -58,6 +58,9 @@ OPERATOR_KEY_MAIN=               # Operator private key used to sign transaction
 # TIER_3_RATE_LIMIT=1600         # Max request limit for static return endpoints
 # LIMIT_DURATION=60000           # Duration in ms for rate limits
 
+# ========== TRANSACTION POOL ========
+# PENDING_TRANSACTION_STORAGE_TTL=30 # Time-to-live (TTL) in seconds for transaction payloads stored in Redis
+
 # ========== HBAR RATE LIMITING ==========
 # HBAR_RATE_LIMIT_TINYBAR=25000000000 # Total HBAR budget (250 HBARs)
 # HBAR_RATE_LIMIT_DURATION=86400000 # HBAR budget limit duration (1 day)

.env.ws.example

@@ -56,6 +56,9 @@ SUBSCRIPTIONS_ENABLED=true       # Must be true for the WebSocket server to func
 # REDIS_RECONNECT_DELAY_MS=1000  # Delay between reconnect retries
 # MULTI_SET=false                # Implementation for setting multiple K/V pairs
 
+# ========== TRANSACTION POOL ========
+# PENDING_TRANSACTION_STORAGE_TTL=30 # Time-to-live (TTL) in seconds for transaction payloads stored in Redis
+
 # ========== OTHER SETTINGS ==========
 # CLIENT_TRANSPORT_SECURITY=false # Enable or disable TLS for both networks
 # USE_ASYNC_TX_PROCESSING=true   # If true, returns tx hash immediately after prechecks

docs/configuration.md

@@ -90,7 +90,6 @@ Unless you need to set a non-default value, it is recommended to only populate o
 | `OPCODELOGGER_ENABLED`                      | "true"                                | Whether the `opcodeLogger` tracer is enabled for a call to `debug_traceTransaction`. This setting should match the value `hiero.mirror.web3.opcode.tracer.enabled` in the Mirror Node used. See [HIP-801](https://hips.hedera.com/hip/hip-801) for more information.                                                                                                                                                                                                                       |
 | `PAYMASTER_ENABLED`                         | "false"                               | Flag to enable or disable the Paymaster functionally                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | `PAYMASTER_WHITELIST`                       | "[]"                                  | List of "to" addresses that will have fully subsidized transactions if the gasPrice was set to 0 by the signer                                                                                                                                                                                                                                                                                                                                                                             |
-| `PENDING_TRANSACTION_STORAGE_TTL`           | 30                                    | Sets the ttl for the pending transactions in the transaction pool
 | `READ_ONLY`                                 | "false"                               | Starts the JSON-RPC Relay in Read-Only mode. In Read-Only mode, write operations, _e.g._, `eth_sendRawTransaction` or query operations to the Consensus Node will trigger an `Unsupported operation` error.                                                                                                                                                                                                                                                                                |
 | `REDIS_ENABLED`                             | "true"                                | Enable usage of Redis as shared cache                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | `REDIS_RECONNECT_DELAY_MS`                  | "1000"                                | Sets the delay between reconnect retries from the Redis client in ms                                                                                                                                                                                                                                                                                                                                                                                                                       |
@@ -102,6 +101,7 @@ Unless you need to set a non-default value, it is recommended to only populate o
 | `TIER_1_RATE_LIMIT`                         | "100"                                 | Maximum restrictive request count limit used for expensive endpoints rate limiting.                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `TIER_2_RATE_LIMIT`                         | "800"                                 | Maximum moderate request count limit used for non expensive endpoints.                                                                                                                                                                                                                                                                                                                                                                                                                     |
 | `TIER_3_RATE_LIMIT`                         | "1600"                                | Maximum relaxed request count limit used for static return endpoints.                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| `PENDING_TRANSACTION_STORAGE_TTL`           | "30"                                  | Time-to-live (TTL) in seconds for transaction payloads stored in Redis. After this period, transaction data expires and is automatically cleaned up by Redis and the Lua-based orphan cleanup mechanism.                                                                                                                                                                                                                                                                                   |
 | `TX_DEFAULT_GAS`                            | "400000"                              | Default gas for transactions that do not specify gas.                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | `USE_ASYNC_TX_PROCESSING`                   | "true"                                | Set to `true` to enable `eth_sendRawTransaction` to return the transaction hash immediately after passing all prechecks, while processing the transaction asynchronously in the background.                                                                                                                                                                                                                                                                                                |
 | `USE_MIRROR_NODE_MODULARIZED_SERVICES`      | null                                  | Controls routing of Mirror Node traffic through modularized services. When set to `true`, enables routing a percentage of traffic to modularized services. When set to `false`, ensures traffic follows the traditional non-modularized flow. When not set (i.e. `null` by default), no specific routing preference is applied. As Mirror Node gradually transitions to a fully modularized architecture across all networks, this setting will eventually default to `true`.              |

packages/config-service/src/services/globalConfig.ts

@@ -519,11 +519,6 @@ const _CONFIG = {
     required: false,
     defaultValue: [],
   },
-  PENDING_TRANSACTION_STORAGE_TTL: {
-    type: 'number',
-    required: false,
-    defaultValue: 30,
-  },
   RATE_LIMIT_DISABLED: {
     type: 'boolean',
     required: false,
@@ -614,6 +609,11 @@ const _CONFIG = {
     required: false,
     defaultValue: false,
   },
+  PENDING_TRANSACTION_STORAGE_TTL: {
+    type: 'number',
+    required: false,
+    defaultValue: 30,
+  },
   TIER_1_RATE_LIMIT: {
     type: 'number',
     required: false,

packages/relay/src/lib/services/ethService/transactionService/TransactionService.ts

@@ -499,7 +499,7 @@ export class TransactionService implements ITransactionService {
 
     // Remove the transaction from the transaction pool after submission
     if (ConfigService.get('ENABLE_TX_POOL')) {
-      await this.transactionPoolService.removeTransaction(originalCallerAddress, parsedTx.hash!);
+      await this.transactionPoolService.removeTransaction(originalCallerAddress, parsedTx.serialized);
     }
 
     sendRawTransactionError = error;

packages/relay/src/lib/services/transactionPoolService/LocalPendingTransactionStorage.ts

@@ -10,11 +10,15 @@ import { PendingTransactionStorage } from '../../types/transactionPool';
  * atomicity across multiple process instances.
  */
 export class LocalPendingTransactionStorage implements PendingTransactionStorage {
-  // Maps address to a Set of transaction hashes for that address
+  // Maps address to a Set of RLP hex payloads for that address
   private readonly pendingTransactions: Map<string, Set<string>>;
 
+  // Global set of all pending RLP hex payloads
+  private readonly globalTransactionIndex: Set<string>;
+
   constructor() {
     this.pendingTransactions = new Map();
+    this.globalTransactionIndex = new Set();
   }
 
   /**
@@ -30,37 +34,44 @@ export class LocalPendingTransactionStorage implements PendingTransactionStorage
 
   /**
    * Adds a pending transaction entry for the given address.
+   * Atomically indexes the transaction (per-address + global) and persists its payload.
    *
    * @param addr - The account address
-   * @param txHash - The transaction hash to add to the pending list
+   * @param rlpHex - The RLP-encoded transaction as a hex string
    */
-  async addToList(addr: string, txHash: string): Promise<void> {
+  async addToList(addr: string, rlpHex: string): Promise<void> {
     // Initialize the set if it doesn't exist
     if (!this.pendingTransactions.has(addr)) {
       this.pendingTransactions.set(addr, new Set());
     }
 
     const addressTransactions = this.pendingTransactions.get(addr)!;
-    addressTransactions.add(txHash);
+    addressTransactions.add(rlpHex);
+
+    // Add to global index
+    this.globalTransactionIndex.add(rlpHex);
   }
 
   /**
    * Removes a transaction from the pending list of the given address.
    *
    * @param address - The account address whose transaction should be removed
-   * @param txHash - The transaction hash to remove
+   * @param rlpHex - The RLP-encoded transaction as a hex string
    */
-  async removeFromList(address: string, txHash: string): Promise<void> {
+  async removeFromList(address: string, rlpHex: string): Promise<void> {
     const addressTransactions = this.pendingTransactions.get(address);
 
     if (addressTransactions) {
-      addressTransactions.delete(txHash);
+      addressTransactions.delete(rlpHex);
 
       // Clean up empty sets to prevent memory leaks
       if (addressTransactions.size === 0) {
         this.pendingTransactions.delete(address);
       }
     }
+
+    // Remove from global index
+    this.globalTransactionIndex.delete(rlpHex);
   }
 
   /**
@@ -70,5 +81,26 @@ export class LocalPendingTransactionStorage implements PendingTransactionStorage
    */
   async removeAll(): Promise<void> {
     this.pendingTransactions.clear();
+    this.globalTransactionIndex.clear();
+  }
+
+  /**
+   * Retrieves all pending transaction payloads (RLP hex) across all addresses.
+   *
+   * @returns Set of all pending transaction RLP hex strings
+   */
+  async getAllTransactionPayloads(): Promise<Set<string>> {
+    return this.globalTransactionIndex;
+  }
+
+  /**
+   * Retrieves pending transaction payloads (RLP hex) for a specific address.
+   *
+   * @param address - The account address to query
+   * @returns Set of transaction RLP hex strings for the address
+   */
+  async getTransactionPayloads(address: string): Promise<Set<string>> {
+    const addressTransactions = this.pendingTransactions.get(address);
+    return addressTransactions ?? new Set();
   }
 }

packages/relay/src/lib/services/transactionPoolService/RedisPendingTransactionStorage.ts

@@ -15,17 +15,29 @@ export class RedisPendingTransactionStorage implements PendingTransactionStorage
    */
   private readonly keyPrefix = 'txpool:pending:';
 
+  /**
+   * Key for storing all pending transactions across all addresses.
+   *
+   * @remarks
+   * This global set contains all pending transaction RLPs from all addresses,
+   * allowing efficient retrieval of the entire pending pool without scanning
+   * individual address keys.
+   */
+  private readonly globalPendingTxsKey = `${this.keyPrefix}global`;
+
   /**
    * The time-to-live (TTL) for the pending transaction storage in seconds.
    */
-  private readonly storageTtl = ConfigService.get('PENDING_TRANSACTION_STORAGE_TTL');
+  private readonly storageTtl: number;
 
   /**
    * Creates a new Redis-backed pending transaction storage.
    *
    * @param redisClient - A connected {@link RedisClientType} instance.
    */
-  constructor(private readonly redisClient: RedisClientType) {}
+  constructor(private readonly redisClient: RedisClientType) {
+    this.storageTtl = ConfigService.get('PENDING_TRANSACTION_STORAGE_TTL');
+  }
 
   /**
    * Resolves the Redis key for a given address.
@@ -38,46 +50,43 @@ export class RedisPendingTransactionStorage implements PendingTransactionStorage
   }
 
   /**
-   * Appends a transaction hash to the pending list for the provided address.
-   *
-   * @remarks
-   * This uses Redis `SADD`, which is atomic. The integer result from Redis is
-   * the new length of the list after the append.
+   * Adds a pending transaction for the given address.
+   * Atomically indexes the transaction (per-address + global) using MULTI/EXEC.
    *
-   * @param addr - Account address whose pending list will be appended to.
-   * @param txHash - Transaction hash to append.
-   * @returns The new pending transaction count after the addition.
+   * @param address - Account address whose pending list will be appended to.
+   * @param rlpHex - The RLP-encoded transaction as a hex string.
    */
-  async addToList(address: string, txHash: string): Promise<void> {
-    const key = this.keyFor(address);
+  async addToList(address: string, rlpHex: string): Promise<void> {
+    const addressKey = this.keyFor(address);
 
-    // doing this to be able to atomically add the transaction hash
-    // and set the expiration time
-    await this.redisClient.multi().sAdd(key, txHash).expire(key, this.storageTtl).execAsPipeline();
+    await this.redisClient
+      .multi()
+      .sAdd(addressKey, rlpHex)
+      .expire(addressKey, this.storageTtl)
+      .sAdd(this.globalPendingTxsKey, rlpHex)
+      .expire(this.globalPendingTxsKey, this.storageTtl)
+      .exec();
   }
 
   /**
-   * Removes a transaction hash from the pending list of the given address.
+   * Removes a transaction from the pending list of the given address.
    *
    * @param address - Account address whose pending list should be modified.
-   * @param txHash - Transaction hash to remove from the list.
+   * @param rlpHex - The RLP-encoded transaction as a hex string.
    */
-  async removeFromList(address: string, txHash: string): Promise<void> {
+  async removeFromList(address: string, rlpHex: string): Promise<void> {
     const key = this.keyFor(address);
-
-    await this.redisClient.sRem(key, txHash);
+    await this.redisClient.multi().sRem(key, rlpHex).sRem(this.globalPendingTxsKey, rlpHex).exec();
   }
 
   /**
    * Removes all keys managed by this storage (all `txpool:pending:*`).
    */
   async removeAll(): Promise<void> {
-    const keys = await this.redisClient.keys(`${this.keyPrefix}*`);
+    const pendingKeys = await this.redisClient.keys(`${this.keyPrefix}*`);
 
-    if (keys.length > 0) {
-      for (const key of keys) {
-        this.redisClient.unlink(key);
-      }
+    if (pendingKeys.length > 0) {
+      await this.redisClient.unlink(pendingKeys);
     }
   }
 
@@ -92,4 +101,26 @@ export class RedisPendingTransactionStorage implements PendingTransactionStorage
 
     return await this.redisClient.sCard(key);
   }
+
+  /**
+   * Retrieves all pending transaction payloads (RLP hex) across all addresses.
+   *
+   * @returns Set of all pending transaction RLP hex strings
+   */
+  async getAllTransactionPayloads(): Promise<Set<string>> {
+    const members = await this.redisClient.sMembers(this.globalPendingTxsKey);
+    return new Set(members);
+  }
+
+  /**
+   * Retrieves pending transaction payloads (RLP hex) for a specific address.
+   *
+   * @param address - The account address to query
+   * @returns Set of transaction RLP hex strings for the address
+   */
+  async getTransactionPayloads(address: string): Promise<Set<string>> {
+    const addressKey = this.keyFor(address);
+    const members = await this.redisClient.sMembers(addressKey);
+    return new Set(members);
+  }
 }