Merge pull request #32 from fastly/kats/improve-kv-store-publish

Improve KV Store Publishing algorithm

Author: harmony7

README.md

@@ -4,7 +4,7 @@ Using a static site generator to build your website? Do you simply need to serve
 
 ## Prerequisites
 
-Node 18 or newer is required during the build step, as we now rely on its `experimental-fetch` feature.
+Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20 or newer.
 
 ## How it works
 

package-lock.json

@@ -41,12 +41,12 @@
     "@fastly/js-compute": "^3.0.0",
     "@types/command-line-args": "^5.2.0",
     "@types/glob-to-regexp": "^0.4.1",
-    "@types/node": "^18.0.0",
+    "@types/node": "^20.0.0",
     "rimraf": "^4.3.0",
     "typescript": "^5.0.2"
   },
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "files": [
     "build",

package.json

@@ -68,7 +68,7 @@ import { applyDefaults } from "../util/data.js";
 import { calculateFileSizeAndHash } from "../util/hash.js";
 import { getFiles } from "../util/files.js";
 import { generateOrLoadPublishId } from "../util/publish-id.js";
-import { FastlyApiContext, loadApiKey } from "../util/fastly-api.js";
+import { FastlyApiContext, FetchError, loadApiKey } from "../util/fastly-api.js";
 import { kvStoreEntryExists, kvStoreSubmitFile } from "../util/kv-store.js";
 import { mergeContentTypes, testFileContentType } from "../../util/content-types.js";
 import { algs } from "../compression/index.js";
@@ -94,6 +94,9 @@ import type {
   ContentFileInfoForWasmInline,
   ContentFileInfoForKVStore,
 } from "../../types/content-assets.js";
+import {
+  attemptWithRetries,
+} from "../util/retryable.js";
 
 type AssetInfo =
   ContentTypeTestResult &
@@ -123,17 +126,54 @@ type KVStoreItemDesc = {
 };
 
 async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) {
-  for (const { kvStoreKey, staticFilePath, text } of kvStoreItems) {
-    if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) {
-      // Already exists in KV Store
-      console.log(`✔️ Asset already exists in KV Store with key "${kvStoreKey}".`)
-    } else {
-      // Upload to KV Store
-      const fileData = fs.readFileSync(staticFilePath);
-      await kvStoreSubmitFile(fastlyApiContext!, kvStoreName!, kvStoreKey, fileData);
-      console.log(`✔️ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`)
+
+  const maxConcurrent = 12;
+  let index = 0; // Shared among workers
+
+  async function worker() {
+    while (index < kvStoreItems.length) {
+      const currentIndex = index;
+      index = index + 1;
+      const { kvStoreKey, staticFilePath, text } = kvStoreItems[currentIndex];
+
+      try {
+        await attemptWithRetries(
+          async() => {
+            if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) {
+              console.log(`✔️ Asset already exists in KV Store with key "${kvStoreKey}".`);
+              return;
+            }
+            const fileData = fs.readFileSync(staticFilePath);
+            await kvStoreSubmitFile(fastlyApiContext, kvStoreName, kvStoreKey, fileData);
+            console.log(`✔️ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`)
+          },
+          {
+            onAttempt(attempt) {
+              if (attempt > 0) {
+                console.log(`Attempt ${attempt + 1} for: ${kvStoreKey}`);
+              }
+            },
+            onRetry(attempt, err, delay) {
+              let statusMessage = 'unknown';
+              if (err instanceof FetchError) {
+                statusMessage = `HTTP ${err.status}`;
+              } else if (err instanceof TypeError) {
+                statusMessage = 'transport';
+              }
+              console.log(`Attempt ${attempt + 1} for ${kvStoreKey} gave retryable error (${statusMessage}), delaying ${delay} ms`);
+            },
+          }
+        );
+      } catch (err) {
+        const e = err instanceof Error ? err : new Error(String(err));
+        console.error(`❌ Failed: ${kvStoreKey} → ${e.message}`);
+        console.error(e.stack);
+      }
     }
   }
+
+  const workers = Array.from({ length: maxConcurrent }, () => worker());
+  await Promise.all(workers);
 }
 
 function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) {
@@ -154,7 +194,7 @@ function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVSt
 
     if (fastlyToml.indexOf(kvStoreName) !== -1) {
       // don't do this!
-      console.error("improperly configured entry for '${kvStoreName}' in fastly.toml");
+      console.error(`improperly configured entry for '${kvStoreName}' in fastly.toml`);
       // TODO: handle thrown exception from callers
       throw "No"!
     }

src/cli/commands/build-static.ts

@@ -543,7 +543,7 @@ ${staticFiles}
       '@fastly/js-compute': '^3.0.0',
     },
     engines: {
-      node: '>=18.0.0',
+      node: '>=20.0.0',
     },
     license: 'UNLICENSED',
     private: true,

src/cli/commands/init-app.ts

@@ -1,4 +1,5 @@
 import { execSync } from 'child_process';
+import { makeRetryable } from './retryable.js';
 
 export interface FastlyApiContext {
   apiToken: string,
@@ -42,7 +43,33 @@ export function loadApiKey(): LoadApiKeyResult | null {
 
 }
 
-export async function callFastlyApi(fastlyApiContext: FastlyApiContext, endpoint: string, queryParams?: URLSearchParams | null, requestInit?: RequestInit): Promise<Response> {
+const RETRYABLE_STATUS_CODES = [
+  408, // Request Timeout
+  409, // Conflict (depends)
+  423, // Locked
+  429, // Too Many Requests
+  500, // Internal Server Error
+  502, // Bad Gateway
+  503, // Service Unavailable
+  504, // Gateway Timeout
+];
+
+export class FetchError extends Error {
+  constructor(message: string, status: number) {
+    super(message);
+    this.name = 'FetchError';
+    this.status = status;
+  }
+  status: number;
+}
+
+export async function callFastlyApi(
+  fastlyApiContext: FastlyApiContext,
+  endpoint: string,
+  operationName: string,
+  queryParams?: URLSearchParams | null,
+  requestInit?: RequestInit,
+): Promise<Response> {
 
   let finalEndpoint = endpoint;
   if (queryParams != null) {
@@ -60,8 +87,24 @@ export async function callFastlyApi(fastlyApiContext: FastlyApiContext, endpoint
   const request = new Request(url, {
     ...requestInit,
     headers,
+    redirect: 'error',
   });
-  const response = await fetch(request);
+  let response;
+  try {
+    response = await fetch(request);
+  } catch(err) {
+    if (err instanceof TypeError) {
+      throw makeRetryable(err);
+    } else {
+      throw err;
+    }
+  }
+  if (!response.ok) {
+    if (!RETRYABLE_STATUS_CODES.includes(response.status)) {
+      throw new FetchError(`${operationName} failed: ${response.status}`, response.status);
+    }
+    throw makeRetryable(new FetchError(`Retryable ${operationName} error: ${response.status}`, response.status));
+  }
   return response;
 
 }