fix(notion-fetch): add final pass validation for missed S3 URLs

Adds a safety net to catch and fix any AWS S3 URLs that persist in the final markdown (e.g. due to callout processing or regex misses), ensuring all expiring URLs are replaced with local images.

Author: luandro

IMAGE_URL_EXPIRATION_SPEC.md

@@ -351,7 +351,48 @@ if (markdownString?.parent) {
 **Complexity**: MEDIUM (requires API integration for full refresh)
 **Risk**: LOW (detection/logging only)
 
-### Phase 3: Monitoring and Metrics (LOW PRIORITY - OPTIONAL/FUTURE WORK)
+### Phase 3: Final Pass Safety Net (HIGH PRIORITY) ⭐
+
+**Goal**: Catch and fix any S3 URLs that remain in the final markdown (e.g., re-introduced by callouts or missed by initial regex)
+
+**Changes**:
+
+1. **Add `validateAndFixRemainingImages` in `imageReplacer.ts`**:
+   - Scans final markdown for any remaining `amazonaws.com` URLs
+   - Uses specific regex to target S3 paths
+   - Re-runs `processAndReplaceImages` if found
+   - Logs warnings if they persist
+
+2. **Call in `processSinglePage`**:
+   - Run this check just before writing the file (after all other processing)
+
+**Specific Code Changes**:
+
+```typescript
+// In imageReplacer.ts
+export async function validateAndFixRemainingImages(markdown, safeFilename) {
+  const s3Regex = /!\[.*?\]\((https:\/\/prod-files-secure\.s3\.[a-z0-9-]+\.amazonaws\.com\/[^\)]+)\)/;
+  if (s3Regex.test(markdown)) {
+    console.warn(`Found S3 URLs in final markdown...`);
+    return processAndReplaceImages(markdown, safeFilename);
+  }
+  return markdown;
+}
+
+// In generateBlocks.ts
+markdownString.parent = await validateAndFixRemainingImages(
+  markdownString.parent,
+  safeFilename
+);
+```
+
+**Benefits**:
+
+- ✅ Catch-all safety net for edge cases
+- ✅ Handles re-introduced URLs from callouts/emojis
+- ✅ Provides final guarantee before file write
+
+### Phase 4: Monitoring and Metrics (LOW PRIORITY - OPTIONAL/FUTURE WORK)
 
 **Status**: NOT IMPLEMENTED - Future enhancement
 

scripts/notion-fetch/__tests__/postProcessing.test.ts

@@ -0,0 +1,102 @@
+
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { processAndReplaceImages, validateAndFixRemainingImages } from "../imageReplacer";
+import { processImageWithFallbacks } from "../imageProcessing";
+
+// Mock dependencies
+vi.mock("../imageProcessing", () => ({
+  processImageWithFallbacks: vi.fn(),
+  createProcessingMetrics: vi.fn(() => ({})),
+  logProcessingMetrics: vi.fn(),
+  logImageFailure: vi.fn(),
+}));
+
+vi.mock("../imageValidation", () => ({
+  validateAndSanitizeImageUrl: vi.fn((url) => ({
+    isValid: true,
+    sanitizedUrl: url,
+  })),
+  createFallbackImageMarkdown: vi.fn((full, url) => full), // Fallback keeps original
+}));
+
+vi.mock("../markdownTransform", () => ({
+  sanitizeMarkdownImages: vi.fn((md) => md),
+}));
+
+vi.mock("../timeoutUtils", () => ({
+  processBatch: vi.fn(async (items, processor) => {
+    // Simple pass-through for testing
+    const results = [];
+    for (const item of items) {
+      results.push({ status: "fulfilled", value: await processor(item) });
+    }
+    return results;
+  }),
+}));
+
+describe("Final Pass Image Validation", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should detect and fix remaining S3 URLs", async () => {
+    const s3Url = "https://prod-files-secure.s3.us-west-2.amazonaws.com/test-image.jpg";
+    const markdown = `Here is an image: ![Alt](${s3Url})`;
+    const safeFilename = "test-page";
+
+    // First pass simulation: mock failure (fallback used)
+    // This simulates the state BEFORE the final pass
+    // In the actual code, processAndReplaceImages returns the markdown.
+    // If it failed, it returned markdown with the original URL.
+
+    // Now we test validateAndFixRemainingImages
+    // We mock processImageWithFallbacks to SUCCEED this time
+    const { processImageWithFallbacks } = await import("../imageProcessing");
+    vi.mocked(processImageWithFallbacks).mockResolvedValue({
+      success: true,
+      newPath: "/images/fixed.jpg",
+      savedBytes: 100,
+      fallbackUsed: false,
+    });
+
+    const result = await validateAndFixRemainingImages(markdown, safeFilename);
+
+    // Expect the URL to be replaced
+    expect(result).toContain("/images/fixed.jpg");
+    expect(result).not.toContain(s3Url);
+    expect(processImageWithFallbacks).toHaveBeenCalled();
+  });
+
+  it("should not modify markdown if no S3 URLs are present", async () => {
+    const markdown = "Here is a local image: ![Alt](/images/local.jpg)";
+    const safeFilename = "test-page";
+
+    const { processImageWithFallbacks } = await import("../imageProcessing");
+
+    const result = await validateAndFixRemainingImages(markdown, safeFilename);
+
+    expect(result).toBe(markdown);
+    expect(processImageWithFallbacks).not.toHaveBeenCalled();
+  });
+
+  it("should handle multiple S3 URLs", async () => {
+    const s3Url1 = "https://prod-files-secure.s3.us-west-2.amazonaws.com/img1.jpg";
+    const s3Url2 = "https://prod-files-secure.s3.us-west-2.amazonaws.com/img2.jpg";
+    const markdown = `![1](${s3Url1}) and ![2](${s3Url2})`;
+    const safeFilename = "test-page";
+
+    const { processImageWithFallbacks } = await import("../imageProcessing");
+    vi.mocked(processImageWithFallbacks).mockImplementation(async (url) => ({
+      success: true,
+      newPath: url.includes("img1") ? "/images/1.jpg" : "/images/2.jpg",
+      savedBytes: 100,
+      fallbackUsed: false,
+    }));
+
+    const result = await validateAndFixRemainingImages(markdown, safeFilename);
+
+    expect(result).toContain("/images/1.jpg");
+    expect(result).toContain("/images/2.jpg");
+    expect(result).not.toContain("amazonaws.com");
+  });
+});

scripts/notion-fetch/generateBlocks.ts

@@ -28,7 +28,10 @@ import { LRUCache, validateCacheSize } from "./cacheStrategies";
 import { getImageCache, logImageFailure } from "./imageProcessing";
 import { setTranslationString, getI18NPath } from "./translationManager";
 import { loadBlocksForPage, loadMarkdownForPage } from "./cacheLoaders";
-import { processAndReplaceImages } from "./imageReplacer";
+import {
+  processAndReplaceImages,
+  validateAndFixRemainingImages,
+} from "./imageReplacer";
 import {
   processToggleSection,
   processHeadingSection,
@@ -329,6 +332,13 @@ async function processSinglePage(
         console.log(chalk.blue(`  ↳ Processed callouts in markdown content`));
       }
 
+      // ✅ FINAL PASS: Check for any AWS S3 URLs that might have been re-introduced
+      // (e.g. by callout processing or missed in the first pass) and fix them.
+      markdownString.parent = await validateAndFixRemainingImages(
+        markdownString.parent,
+        safeFilename
+      );
+
       // Sanitize content to fix malformed HTML/JSX tags
       markdownString.parent = sanitizeMarkdownContent(markdownString.parent);
 

scripts/notion-fetch/imageReplacer.ts

@@ -361,3 +361,50 @@ export async function processAndReplaceImages(
     metrics,
   };
 }
+
+/**
+ * Validates final markdown for remaining S3 URLs and attempts to fix them.
+ * This acts as a safety net for images missed by the initial pass or re-introduced
+ * by subsequent processing (e.g. callouts).
+ *
+ * @param markdown - The final markdown content to check
+ * @param safeFilename - Safe filename for logging
+ * @returns The processed markdown (potentially with fixes applied)
+ */
+export async function validateAndFixRemainingImages(
+  markdown: string,
+  safeFilename: string
+): Promise<string> {
+  // Regex for AWS S3 URLs in markdown image syntax
+  // Matches: ![alt](https://prod-files-secure.s3...amazonaws.com/...)
+  const s3Regex =
+    /!\[.*?\]\((https:\/\/prod-files-secure\.s3\.[a-z0-9-]+\.amazonaws\.com\/[^\)]+)\)/;
+
+  if (!s3Regex.test(markdown)) {
+    return markdown;
+  }
+
+  console.warn(
+    chalk.yellow(
+      `⚠️  Found AWS S3 URLs in final markdown for ${safeFilename}. Running final replacement pass...`
+    )
+  );
+
+  // Re-run processAndReplaceImages
+  const result = await processAndReplaceImages(markdown, safeFilename);
+
+  // Check if any remain (indicating persistent failure)
+  if (s3Regex.test(result.markdown)) {
+    console.warn(
+      chalk.red(
+        `❌ Failed to replace all S3 URLs in final pass for ${safeFilename}. Some images may expire.`
+      )
+    );
+  } else {
+    console.info(
+      chalk.green(`✅ Successfully fixed remaining S3 URLs in ${safeFilename}`)
+    );
+  }
+
+  return result.markdown;
+}