Merge branch 'main' into feat/tb-partitioned-activityRelations

Author: epipav

backend/config/custom-environment-variables.json

@@ -84,10 +84,6 @@
     "appId": "CROWD_SLACK_APP_ID",
     "appToken": "CROWD_SLACK_APP_TOKEN"
   },
-  "slackNotifier": {
-    "clientId": "CROWD_SLACK_NOTIFIER_CLIENT_ID",
-    "clientSecret": "CROWD_SLACK_NOTIFIER_CLIENT_SECRET"
-  },
   "google": {
     "clientId": "CROWD_GOOGLE_CLIENT_ID",
     "clientSecret": "CROWD_GOOGLE_CLIENT_SECRET",

backend/config/default.json

@@ -26,7 +26,6 @@
   "slack": {
     "maxRetrospectInSeconds": 3600
   },
-  "slackNotifier": {},
   "google": {},
   "discord": {
     "maxRetrospectInSeconds": 3600

backend/src/database/migrations/U1764675824__addStuckReOnboardColumnToGitRepositories.sql

@@ -0,0 +1,6 @@
+-- Remove stuckRequiresReOnboard column from git.repositories table
+-- Rollback for V1764675824__addStuckReOnboardColumnToGitRepositories.sql
+
+ALTER TABLE git.repositories 
+DROP COLUMN IF EXISTS "stuckRequiresReOnboard";
+

backend/src/database/migrations/V1764675824__addStuckReOnboardColumnToGitRepositories.sql

@@ -0,0 +1,9 @@
+-- Add stuckRequiresReOnboard column to git.repositories table
+-- This column indicates if a repository is stuck and requires re-onboarding
+
+ALTER TABLE git.repositories 
+ADD COLUMN "stuckRequiresReOnboard" BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- Add comment for documentation
+COMMENT ON COLUMN git.repositories."stuckRequiresReOnboard" IS 'Indicates if the stuck repository is resolved by a re-onboarding';
+

backend/src/services/integrationService.ts

@@ -26,7 +26,7 @@ import {
   startNangoSync,
 } from '@crowd/nango'
 import { RedisCache } from '@crowd/redis'
-import { WorkflowIdReusePolicy } from '@crowd/temporal'
+import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal'
 import { CodePlatform, Edition, PlatformType } from '@crowd/types'
 
 import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions'
@@ -919,11 +919,12 @@ export default class IntegrationService {
       await this.options.temporal.workflow.start('syncGithubIntegration', {
         taskQueue: 'nango',
         workflowId: `github-nango-sync/${integration.id}`,
-        workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE,
+        workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
+        workflowIdConflictPolicy: WorkflowIdConflictPolicy.USE_EXISTING,
         retry: {
           maximumAttempts: 10,
         },
-        args: [{ integrationIds: [integration.id] }],
+        args: [{ integrationId: integration.id }],
       })
 
       return await this.findById(integration.id)

frontend/src/modules/admin/modules/insights-projects/widgets.ts

@@ -26,6 +26,10 @@ export enum Widgets {
   AVERAGE_TIME_TO_MERGE = 'averageTimeToMerge',
   WAIT_TIME_FOR_1ST_REVIEW = 'waitTimeFor1stReview',
   CODE_REVIEW_ENGAGEMENT = 'codeReviewEngagement',
+  REVIEW_EFFICIENCY = 'reviewEfficiency',
+  PATCHSET_PER_REVIEW = 'patchsetPerReview',
+  MEDIAN_TIME_TO_MERGE = 'medianTimeToMerge',
+  MEDIAN_TIME_TO_REVIEW = 'medianTimeToReview',
 }
 
 export const WIDGETS_GROUPS = [
@@ -151,6 +155,22 @@ export const WIDGETS_GROUPS = [
         name: 'Code review engagement',
         key: Widgets.CODE_REVIEW_ENGAGEMENT,
       },
+      {
+        name: 'Review efficiency',
+        key: Widgets.REVIEW_EFFICIENCY,
+      },
+      {
+        name: 'Patchsets per review',
+        key: Widgets.PATCHSET_PER_REVIEW,
+      },
+      {
+        name: 'Median time to merge',
+        key: Widgets.MEDIAN_TIME_TO_MERGE,
+      },
+      {
+        name: 'Median time to review',
+        key: Widgets.MEDIAN_TIME_TO_REVIEW,
+      },
     ],
   },
 ];

pnpm-lock.yaml

@@ -17,6 +17,7 @@
     "@crowd/data-access-layer": "workspace:*",
     "@crowd/logging": "workspace:*",
     "@crowd/nango": "workspace:*",
+    "@crowd/slack": "workspace:*",
     "@crowd/temporal": "workspace:*",
     "@crowd/types": "workspace:*",
     "@crowd/common_services": "workspace:*",

services/apps/cron_service/package.json

@@ -1,11 +1,16 @@
 import CronTime from 'cron-time-generator'
 
-import { IS_DEV_ENV } from '@crowd/common'
+import { ConcurrencyLimiter, IS_DEV_ENV } from '@crowd/common'
 import { READ_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database'
 import { fetchNangoIntegrationData } from '@crowd/data-access-layer/src/integrations'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { NangoIntegration, nangoIntegrationToPlatform } from '@crowd/nango'
-import { TEMPORAL_CONFIG, WorkflowIdReusePolicy, getTemporalClient } from '@crowd/temporal'
+import {
+  TEMPORAL_CONFIG,
+  WorkflowIdConflictPolicy,
+  WorkflowIdReusePolicy,
+  getTemporalClient,
+} from '@crowd/temporal'
 import { PlatformType } from '@crowd/types'
 
 import { IJobDefinition } from '../types'
@@ -15,7 +20,7 @@ const job: IJobDefinition = {
   cronTime: CronTime.every(
     Number(process.env.CROWD_GH_NANGO_SYNC_INTERVAL_MINUTES || IS_DEV_ENV ? 5 : 60),
   ).minutes(),
-  timeout: 10 * 60,
+  timeout: 4 * 60 * 60, // 4 hours
   process: async (ctx) => {
     ctx.log.info('Triggering nango API check as if a webhook was received!')
 
@@ -27,9 +32,13 @@ const job: IJobDefinition = {
       nangoIntegrationToPlatform(NangoIntegration.GITHUB),
     ])
 
-    const ids: string[] = []
+    const limiter = new ConcurrencyLimiter(5)
 
-    for (const int of integrations) {
+    // Collect all workflow start operations
+    const workflowStarts: Array<() => Promise<void>> = []
+
+    for (let i = 0; i < integrations.length; i++) {
+      const int = integrations[i]
       const { id, platform } = int
 
       if (platform !== PlatformType.GITHUB_NANGO) {
@@ -40,22 +49,51 @@ const job: IJobDefinition = {
         {
           integrationId: id,
         },
-        'Triggering nango github integration sync!',
+        `${i + 1}/${integrations.length} Triggering nango github integration sync!`,
       )
 
-      ids.push(id)
+      workflowStarts.push(async () => {
+        await temporal.workflow
+          .start('syncGithubIntegration', {
+            taskQueue: 'nango',
+            workflowId: `github-nango-sync/cron/${id}`,
+            workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
+            workflowIdConflictPolicy: WorkflowIdConflictPolicy.USE_EXISTING,
+            retry: {
+              maximumAttempts: 10,
+            },
+            args: [{ integrationId: id }],
+          })
+          .catch((err) =>
+            ctx.log.error(err, 'Error while triggering nango github integration sync!'),
+          )
+      })
     }
 
-    await temporal.workflow.start('syncGithubIntegration', {
-      taskQueue: 'nango',
-      workflowId: `github-nango-sync/cron/${new Date().toISOString()}`,
-      workflowIdReusePolicy:
-        WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE_FAILED_ONLY,
-      retry: {
-        maximumAttempts: 10,
-      },
-      args: [{ integrationIds: ids }],
+    ctx.log.info(
+      `Triggering nango github integration syncs with ${workflowStarts.length} workflows!`,
+    )
+
+    // Track completed workflows
+    let completedWorkflows = 0
+
+    // Register callback to track completed workflows
+    limiter.setOnJobComplete(() => {
+      completedWorkflows++
+      if (completedWorkflows % 100 === 0) {
+        ctx.log.info(`Triggered ${completedWorkflows} nango github integration syncs so far...`)
+      }
     })
+
+    // Process all workflow starts with concurrency limit
+    for (const workflowStart of workflowStarts) {
+      await limiter.schedule(workflowStart)
+    }
+
+    // Wait for all remaining jobs to complete
+    await limiter.waitForFinish()
+
+    ctx.log.info(`Triggered ${completedWorkflows} nango github integration syncs in total`)
   },
 }