feat: add principal cache etag to account endpoints (#2097)

* feat: add principal cache

* fix: also consider sponsor transactions

* chore: import sponsors in migration

* fix: add down migration

Author: rafaelcr

migrations/1727465879167_principal-stx-txs-sponsors.js

@@ -0,0 +1,21 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined;
+
+exports.up = pgm => {
+  pgm.sql(`
+    INSERT INTO principal_stx_txs
+      (principal, tx_id, block_height, index_block_hash, microblock_hash, microblock_sequence,
+      tx_index, canonical, microblock_canonical)
+    (
+      SELECT
+        sponsor_address AS principal, tx_id, block_height, index_block_hash, microblock_hash,
+        microblock_sequence, tx_index, canonical, microblock_canonical
+      FROM txs
+      WHERE sponsor_address IS NOT NULL
+    )
+    ON CONFLICT ON CONSTRAINT unique_principal_tx_id_index_block_hash_microblock_hash DO NOTHING  
+  `);
+};
+
+exports.down = pgm => {};

src/api/controllers/cache-controller.ts

@@ -2,17 +2,13 @@ import * as prom from 'prom-client';
 import { normalizeHashString } from '../../helpers';
 import { PgStore } from '../../datastore/pg-store';
 import { logger } from '../../logger';
-import { sha256 } from '@hirosystems/api-toolkit';
+import {
+  CACHE_CONTROL_MUST_REVALIDATE,
+  parseIfNoneMatchHeader,
+  sha256,
+} from '@hirosystems/api-toolkit';
 import { FastifyReply, FastifyRequest } from 'fastify';
 
-/**
- * A `Cache-Control` header used for re-validation based caching.
- * * `public` == allow proxies/CDNs to cache as opposed to only local browsers.
- * * `no-cache` == clients can cache a resource but should revalidate each time before using it.
- * * `must-revalidate` == somewhat redundant directive to assert that cache must be revalidated, required by some CDNs
- */
-const CACHE_CONTROL_MUST_REVALIDATE = 'public, no-cache, must-revalidate';
-
 /**
  * Describes a key-value to be saved into a request's locals, representing the current
  * state of the chain depending on the type of information being requested by the endpoint.
@@ -25,6 +21,8 @@ enum ETagType {
   mempool = 'mempool',
   /** ETag based on the status of a single transaction across the mempool or canonical chain. */
   transaction = 'transaction',
+  /** Etag based on the confirmed balance of a single principal (STX address or contract id) */
+  principal = 'principal',
 }
 
 /** Value that means the ETag did get calculated but it is empty. */
@@ -75,52 +73,6 @@ function getETagMetrics(): ETagCacheMetrics {
   return _eTagMetrics;
 }
 
-/**
- * Parses the etag values from a raw `If-None-Match` request header value.
- * The wrapping double quotes (if any) and validation prefix (if any) are stripped.
- * The parsing is permissive to account for commonly non-spec-compliant clients, proxies, CDNs, etc.
- * E.g. the value:
- * ```js
- * `"a", W/"b", c,d,   "e", "f"`
- * ```
- * Would be parsed and returned as:
- * ```js
- * ['a', 'b', 'c', 'd', 'e', 'f']
- * ```
- * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#syntax
- * ```
- * If-None-Match: "etag_value"
- * If-None-Match: "etag_value", "etag_value", ...
- * If-None-Match: *
- * ```
- * @param ifNoneMatchHeaderValue - raw header value
- * @returns an array of etag values
- */
-export function parseIfNoneMatchHeader(
-  ifNoneMatchHeaderValue: string | undefined
-): string[] | undefined {
-  if (!ifNoneMatchHeaderValue) {
-    return undefined;
-  }
-  // Strip wrapping double quotes like `"hello"` and the ETag validation-prefix like `W/"hello"`.
-  // The API returns compliant, strong-validation ETags (double quoted ASCII), but can't control what
-  // clients, proxies, CDNs, etc may provide.
-  const normalized = /^(?:"|W\/")?(.*?)"?$/gi.exec(ifNoneMatchHeaderValue.trim())?.[1];
-  if (!normalized) {
-    // This should never happen unless handling a buggy request with something like `If-None-Match: ""`,
-    // or if there's a flaw in the above code. Log warning for now.
-    logger.warn(`Normalized If-None-Match header is falsy: ${ifNoneMatchHeaderValue}`);
-    return undefined;
-  } else if (normalized.includes(',')) {
-    // Multiple etag values provided, likely irrelevant extra values added by a proxy/CDN.
-    // Split on comma, also stripping quotes, weak-validation prefixes, and extra whitespace.
-    return normalized.split(/(?:W\/"|")?(?:\s*),(?:\s*)(?:W\/"|")?/gi);
-  } else {
-    // Single value provided (the typical case)
-    return [normalized];
-  }
-}
-
 async function calculateETag(
   db: PgStore,
   etagType: ETagType,
@@ -155,7 +107,7 @@ async function calculateETag(
         }
         return digest.result.digest;
       } catch (error) {
-        logger.error(error, 'Unable to calculate mempool');
+        logger.error(error, 'Unable to calculate mempool etag');
         return;
       }
 
@@ -178,7 +130,20 @@ async function calculateETag(
         ];
         return sha256(elements.join(':'));
       } catch (error) {
-        logger.error(error, 'Unable to calculate transaction');
+        logger.error(error, 'Unable to calculate transaction etag');
+        return;
+      }
+
+    case ETagType.principal:
+      try {
+        const params = req.params as { address?: string; principal?: string };
+        const principal = params.address ?? params.principal;
+        if (!principal) return ETAG_EMPTY;
+        const activity = await db.getPrincipalLastActivityTxIds(principal);
+        const text = `${activity.stx_tx_id}:${activity.ft_tx_id}:${activity.nft_tx_id}`;
+        return sha256(text);
+      } catch (error) {
+        logger.error(error, 'Unable to calculate principal etag');
         return;
       }
   }
@@ -224,3 +189,7 @@ export async function handleMempoolCache(request: FastifyRequest, reply: Fastify
 export async function handleTransactionCache(request: FastifyRequest, reply: FastifyReply) {
   return handleCache(ETagType.transaction, request, reply);
 }
+
+export async function handlePrincipalCache(request: FastifyRequest, reply: FastifyReply) {
+  return handleCache(ETagType.principal, request, reply);
+}

src/api/routes/address.ts

@@ -15,7 +15,12 @@ import {
 } from '../controllers/db-controller';
 import { InvalidRequestError, InvalidRequestErrorType, NotFoundError } from '../../errors';
 import { decodeClarityValueToRepr } from 'stacks-encoding-native-js';
-import { handleChainTipCache, handleMempoolCache } from '../controllers/cache-controller';
+import {
+  handleChainTipCache,
+  handleMempoolCache,
+  handlePrincipalCache,
+  handleTransactionCache,
+} from '../controllers/cache-controller';
 import { PgStore } from '../../datastore/pg-store';
 import { logger } from '../../logger';
 import { has0xPrefix } from '@hirosystems/api-toolkit';
@@ -86,7 +91,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/stx',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         operationId: 'get_account_stx_balance',
         summary: 'Get account STX balance',
@@ -142,7 +147,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/balances',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         operationId: 'get_account_balance',
         summary: 'Get account balances',
@@ -234,7 +239,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/transactions',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         deprecated: true,
         operationId: 'get_account_transactions',
@@ -307,7 +312,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/:tx_id/with_transfers',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handleTransactionCache,
       schema: {
         deprecated: true,
         operationId: 'get_single_transaction_with_transfers',
@@ -373,7 +378,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/transactions_with_transfers',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         deprecated: true,
         operationId: 'get_account_transactions_with_transfers',
@@ -485,7 +490,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/assets',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         operationId: 'get_account_assets',
         summary: 'Get account assets',
@@ -533,7 +538,7 @@ export const AddressRoutes: FastifyPluginAsync<
   fastify.get(
     '/:principal/stx_inbound',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         operationId: 'get_account_inbound',
         summary: 'Get inbound STX transfers',

src/api/routes/v2/addresses.ts

@@ -1,4 +1,7 @@
-import { handleChainTipCache } from '../../../api/controllers/cache-controller';
+import {
+  handlePrincipalCache,
+  handleTransactionCache,
+} from '../../../api/controllers/cache-controller';
 import { AddressParamsSchema, AddressTransactionParamsSchema } from './schemas';
 import { parseDbAddressTransactionTransfer, parseDbTxWithAccountTransferSummary } from './helpers';
 import { InvalidRequestError, NotFoundError } from '../../../errors';
@@ -23,7 +26,7 @@ export const AddressRoutesV2: FastifyPluginAsync<
   fastify.get(
     '/:address/transactions',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handlePrincipalCache,
       schema: {
         operationId: 'get_address_transactions',
         summary: 'Get address transactions',
@@ -71,7 +74,7 @@ export const AddressRoutesV2: FastifyPluginAsync<
   fastify.get(
     '/:address/transactions/:tx_id/events',
     {
-      preHandler: handleChainTipCache,
+      preHandler: handleTransactionCache,
       schema: {
         operationId: 'get_address_transaction_events',
         summary: 'Get events for an address transaction',

src/datastore/pg-store.ts

@@ -4404,4 +4404,44 @@ export class PgStore extends BasePgStore {
     }
     return result;
   }
+
+  /** Retrieves the last transaction IDs with STX, FT and NFT activity for a principal */
+  async getPrincipalLastActivityTxIds(
+    principal: string
+  ): Promise<{ stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }> {
+    const result = await this.sql<
+      { stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }[]
+    >`
+      WITH last_stx AS (
+        SELECT tx_id
+        FROM principal_stx_txs
+        WHERE principal = ${principal} AND canonical = true AND microblock_canonical = true
+        ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC
+        LIMIT 1
+      ),
+      last_ft AS (
+        SELECT tx_id
+        FROM ft_events
+        WHERE (sender = ${principal} OR recipient = ${principal})
+          AND canonical = true
+          AND microblock_canonical = true
+        ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
+        LIMIT 1
+      ),
+      last_nft AS (
+        SELECT tx_id
+        FROM nft_events
+        WHERE (sender = ${principal} OR recipient = ${principal})
+          AND canonical = true
+          AND microblock_canonical = true
+        ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
+        LIMIT 1
+      )
+      SELECT
+        (SELECT tx_id FROM last_stx) AS stx_tx_id,
+        (SELECT tx_id FROM last_ft) AS ft_tx_id,
+        (SELECT tx_id FROM last_nft) AS nft_tx_id
+    `;
+    return result[0];
+  }
 }

src/datastore/pg-write-store.ts

@@ -1180,6 +1180,7 @@ export class PgWriteStore extends PgStore {
           tx.token_transfer_recipient_address,
           tx.contract_call_contract_id,
           tx.smart_contract_contract_id,
+          tx.sponsor_address,
         ].filter((p): p is string => !!p)
       );
       for (const event of stxEvents) {

src/event-replay/parquet-based/importers/new-block-importer.ts

@@ -251,6 +251,7 @@ const populateBatchInserters = (db: PgWriteStore) => {
             entry.tx.token_transfer_recipient_address,
             entry.tx.contract_call_contract_id,
             entry.tx.smart_contract_contract_id,
+            entry.tx.sponsor_address,
           ]
             .filter((p): p is string => !!p)
             .forEach(p => principals.add(p));