fix: redis graceful exit and job cleanup (wei#636)

* fix: redis graceful exit and job cleanup

* release: v2.0.0-alpha.4

Author: wei

deno.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.0.0-alpha.3",
+  "version": "2.0.0-alpha.4",
   "tasks": {
     "dev": "deno run --env-file --allow-all src/index.ts",
     "dev:skip-full-sync": "deno task dev --skip-full-sync",

scripts/full-sync.ts

@@ -8,10 +8,11 @@ import { appConfig } from "@/src/configs/app-config.ts";
 
 async function main() {
   let exitCode = 0;
+  let redisClient: ReturnType<typeof getRedisClient> | undefined;
 
   try {
     await connectMongoDB();
-    const redisClient = getRedisClient(`${appConfig.appSlug}-full-sync`);
+    redisClient = getRedisClient(`${appConfig.appSlug}-full-sync`);
 
     const probot = createProbot({ overrides: { log: logger } });
     await fullSync(probot, {
@@ -23,6 +24,13 @@ async function main() {
     exitCode = 1;
   } finally {
     await disconnectMongoDB();
+    if (redisClient) {
+      try {
+        await redisClient.quit();
+      } catch {
+        // ignore
+      }
+    }
     Deno.exit(exitCode);
   }
 }

src/index.ts

@@ -56,6 +56,12 @@ Deno.addSignalListener("SIGTERM", () => handleAppTermination("SIGTERM"));
 function handleAppTermination(signal: string) {
   log.info(`[${signal}] Signal received: closing MongoDB connection`);
   disconnectMongoDB();
+  try {
+    // Close Redis connection to avoid lingering connections
+    redisClient.quit();
+  } catch {
+    // ignore
+  }
   log.info("[MongoDB] Connection closed due to app termination");
   Deno.exit(0);
 }

src/worker.ts

@@ -12,11 +12,22 @@ const redisClient = new Redis(appConfig.redisConfig!, {
   name: `${appConfig.appSlug}-worker`,
 });
 
+const MAX_RETAINED_JOBS = 1000;
+const JOB_RETENTION_SECONDS = 3600; // 1 hour
+
 const worker = createSchedulerWorker(
   RepoJobProcessor,
   {
     connection: redisClient,
     concurrency: 10,
+    removeOnComplete: {
+      count: MAX_RETAINED_JOBS,
+      age: JOB_RETENTION_SECONDS,
+    },
+    removeOnFail: {
+      count: MAX_RETAINED_JOBS,
+      age: JOB_RETENTION_SECONDS,
+    },
   },
 );
 
@@ -31,6 +42,11 @@ worker.on("failed", (job, err) => {
 const gracefulShutdown = async (signal: string) => {
   console.log(`Received ${signal}, closing worker...`);
   await worker.close();
+  try {
+    await redisClient.quit();
+  } catch {
+    // ignore
+  }
   Deno.exit(0);
 };