From fdd321175c5c83fc555b152c767acec15de53b5b Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 07:57:58 +0800 Subject: [PATCH 01/12] refactor(collab-service): move ws to the same port as http --- .../Dockerfile.collaboration | 2 +- backend/collaboration-service/src/config.js | 3 +-- .../collaboration-service/src/http/httpServer.js | 4 ++-- backend/collaboration-service/src/server.js | 2 +- .../src/websocket/websocketServer.js | 9 +++++---- docker-compose.yml | 7 ++----- .../components/practice/code-editor-panel.tsx | 4 ++-- frontend/src/lib/api-client.ts | 16 ++++++++-------- frontend/src/lib/api-config.ts | 6 ++---- frontend/src/stores/collaboration-store.ts | 15 ++++++--------- frontend/src/utils/config.ts | 11 ----------- 11 files changed, 30 insertions(+), 49 deletions(-) delete mode 100644 frontend/src/utils/config.ts diff --git a/backend/collaboration-service/Dockerfile.collaboration b/backend/collaboration-service/Dockerfile.collaboration index d3fbbde204..a8dcff2c5e 100644 --- a/backend/collaboration-service/Dockerfile.collaboration +++ b/backend/collaboration-service/Dockerfile.collaboration @@ -28,7 +28,7 @@ RUN chown -R collaboration-service:nodejs /app USER collaboration-service # Expose ports for HTTP and WebSocket servers -EXPOSE 8004 8005 +EXPOSE 8004 # Start the application CMD ["pnpm", "start"] \ No newline at end of file diff --git a/backend/collaboration-service/src/config.js b/backend/collaboration-service/src/config.js index 8a53a589f4..2fc0a5641a 100644 --- a/backend/collaboration-service/src/config.js +++ b/backend/collaboration-service/src/config.js @@ -1,7 +1,6 @@ export const config = { // Server Ports - WS_PORT: process.env.WS_PORT || 8005, - HTTP_PORT: process.env.HTTP_PORT || 8004, + PORT: process.env.PORT || 8004, // Database MONGO_URI: process.env.MONGODB_URI || diff --git a/backend/collaboration-service/src/http/httpServer.js b/backend/collaboration-service/src/http/httpServer.js index 7b231c6c80..0d59f141ee 100644 --- a/backend/collaboration-service/src/http/httpServer.js +++ b/backend/collaboration-service/src/http/httpServer.js @@ -58,12 +58,12 @@ class HttpServer { start() { return new Promise((resolve, reject) => { this.server = http.createServer(this.app); - this.server.listen(config.HTTP_PORT, (error) => { + this.server.listen(config.PORT, (error) => { if (error) { console.error("Failed to start HTTP server:", error); reject(error); } else { - console.log(`HTTP API server running on http://localhost:${config.HTTP_PORT}`); + console.log(`HTTP API server running on http://localhost:${config.PORT}`); resolve(this.server); } }); diff --git a/backend/collaboration-service/src/server.js b/backend/collaboration-service/src/server.js index 5fd1b183c5..00807fc04c 100644 --- a/backend/collaboration-service/src/server.js +++ b/backend/collaboration-service/src/server.js @@ -16,7 +16,7 @@ async function startServer() { await httpServer.start(); // Step 4: Start WebSocket server - await webSocketServer.start(); + await webSocketServer.start(httpServer.server); // Step 4: Start Kafka consumer for room creation await setupRoomCreationConsumer(); diff --git a/backend/collaboration-service/src/websocket/websocketServer.js b/backend/collaboration-service/src/websocket/websocketServer.js index e617b62756..52051383dc 100644 --- a/backend/collaboration-service/src/websocket/websocketServer.js +++ b/backend/collaboration-service/src/websocket/websocketServer.js @@ -11,14 +11,15 @@ class WebSocketServerManager { } /** - * Start the WebSocket server + * Start the WebSocket server by attaching to an existing HTTP server + * @param {http.Server} httpServer - The HTTP server instance from Express */ - start() { + start(httpServer) { return new Promise((resolve, reject) => { try { - this.wss = new WebSocketServer({ port: config.WS_PORT }); + this.wss = new WebSocketServer({ server: httpServer }); this.wss.on("connection", this.handleConnection.bind(this)); - console.log(`WebSocket server running on ws://localhost:${config.WS_PORT}`); + console.log(`WebSocket server is attached to HTTP server on port ${config.PORT}`); resolve(this.wss); } catch (error) { console.error("Failed to start WebSocket server:", error); diff --git a/docker-compose.yml b/docker-compose.yml index 2a910941a8..8eab1de753 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,14 +139,12 @@ services: restart: always ports: - '8004:8004' - - '8005:8005' env_file: - ./backend/collaboration-service/.env environment: - MONGODB_URI=mongodb://admin:password@mongodb:27017/peerprepCollabServiceDB?authSource=admin - WEB_BASE_URL=http://localhost:3000 - - HTTP_PORT=8004 - - WS_PORT=8005 + - PORT=8004 - ROOM_TIMEOUT_MINUTES=10 - QUESTION_SERVICE_URL=http://question-service:8003 - KAFKA_HOST=kafka @@ -245,8 +243,7 @@ services: - NEXT_PUBLIC_USER_SERVICE_URL=http://localhost:8001 - NEXT_PUBLIC_MATCHING_SERVICE_URL=http://localhost:8002 - NEXT_PUBLIC_QUESTION_SERVICE_URL=http://localhost:8003 - - NEXT_PUBLIC_COLLABORATION_SERVICE_HTTP_URL=http://localhost:8004 - - NEXT_PUBLIC_COLLABORATION_SERVICE_WS_URL=ws://localhost:8005 + - NEXT_PUBLIC_COLLABORATION_SERVICE_URL=http://localhost:8004 depends_on: - user-service - collaboration-service diff --git a/frontend/src/components/practice/code-editor-panel.tsx b/frontend/src/components/practice/code-editor-panel.tsx index 91e4577818..9715281bfe 100644 --- a/frontend/src/components/practice/code-editor-panel.tsx +++ b/frontend/src/components/practice/code-editor-panel.tsx @@ -21,7 +21,7 @@ import { } from "@/components/ui/select"; import { useCollaborationState, useCollaborationActions } from "@/stores/collaboration-store"; import { ProgrammingLanguage, ProgrammingLanguageDisplay, ConnectionState } from "@/utils/enums"; -import { collaborationConfig } from "@/utils/config"; +import { getCollaborationURL } from "@/lib/api-config"; interface CodeEditorPanelProps { readOnly?: boolean; @@ -82,7 +82,7 @@ export default function CodeEditorPanel({ readOnly = false }: CodeEditorPanelPro ydocRef.current = ydoc; // Create WebSocket provider with token in URL - const wsUrl = collaborationConfig.WS_URL; + const wsUrl = getCollaborationURL().replace(/^http/, "ws"); const roomWithToken = `${roomId}?token=${accessToken}`; const provider = new WebsocketProvider(wsUrl, roomWithToken, ydoc); providerRef.current = provider; diff --git a/frontend/src/lib/api-client.ts b/frontend/src/lib/api-client.ts index 921d3a3f86..b87b780a6b 100644 --- a/frontend/src/lib/api-client.ts +++ b/frontend/src/lib/api-client.ts @@ -164,7 +164,7 @@ const userServiceClient = axios.create({ // Create axios instance for collaboration service const collaborationServiceClient = axios.create({ - baseURL: apiConfig.collaborationService.httpURL, + baseURL: apiConfig.collaborationService.baseURL, withCredentials: true, }); @@ -223,10 +223,10 @@ export const authRequest = async ( let client = userServiceClient; // If URL is a full URL and matches collaboration service, use collaboration client - if (url.startsWith(apiConfig.collaborationService.httpURL)) { + if (url.startsWith(apiConfig.collaborationService.baseURL)) { client = collaborationServiceClient; // Remove base URL since client already has it - config.url = url.replace(apiConfig.collaborationService.httpURL, ""); + config.url = url.replace(apiConfig.collaborationService.baseURL, ""); } else if (url.startsWith(apiConfig.questionService.baseURL)) { client = questionServiceClient; config.url = url.replace(apiConfig.questionService.baseURL, ""); @@ -246,9 +246,9 @@ export const authRequest = async ( const url = config.url || ""; let client = userServiceClient; - if (url.startsWith(apiConfig.collaborationService.httpURL)) { + if (url.startsWith(apiConfig.collaborationService.baseURL)) { client = collaborationServiceClient; - config.url = url.replace(apiConfig.collaborationService.httpURL, ""); + config.url = url.replace(apiConfig.collaborationService.baseURL, ""); } else if (url.startsWith(apiConfig.questionService.baseURL)) { client = questionServiceClient; config.url = url.replace(apiConfig.questionService.baseURL, ""); @@ -343,7 +343,7 @@ export const collaborationAPI = { getRoomDetails: async (roomId: string): Promise => { const res = await authRequest({ method: "GET", - url: `${apiConfig.collaborationService.httpURL}/api/v1/rooms/${roomId}`, + url: `${apiConfig.collaborationService.baseURL}/api/v1/rooms/${roomId}`, }); return res.data; }, @@ -351,7 +351,7 @@ export const collaborationAPI = { getUserRooms: async (userId: string): Promise => { const res = await authRequest({ method: "GET", - url: `${apiConfig.collaborationService.httpURL}/api/v1/rooms?userId=${userId}`, + url: `${apiConfig.collaborationService.baseURL}/api/v1/rooms?userId=${userId}`, }); return res.data; }, @@ -359,7 +359,7 @@ export const collaborationAPI = { changeLanguage: async (roomId: string, language: string): Promise => { const res = await authRequest({ method: "PATCH", - url: `${apiConfig.collaborationService.httpURL}/api/v1/rooms/${roomId}/language`, + url: `${apiConfig.collaborationService.baseURL}/api/v1/rooms/${roomId}/language`, data: { language }, }); return res.data; diff --git a/frontend/src/lib/api-config.ts b/frontend/src/lib/api-config.ts index 9ac7c49d44..8cc14c8be5 100644 --- a/frontend/src/lib/api-config.ts +++ b/frontend/src/lib/api-config.ts @@ -9,13 +9,11 @@ export const apiConfig = { baseURL: process.env.NEXT_PUBLIC_QUESTION_SERVICE_URL || "http://localhost:8003", }, collaborationService: { - httpURL: process.env.NEXT_PUBLIC_COLLABORATION_SERVICE_HTTP_URL || "http://localhost:8004", - wsURL: process.env.NEXT_PUBLIC_COLLABORATION_SERVICE_WS_URL || "ws://localhost:8005", + baseURL: process.env.NEXT_PUBLIC_COLLABORATION_SERVICE_URL || "http://localhost:8004", }, } as const; export const getUserServiceURL = () => apiConfig.userService.baseURL; export const getMatchingServiceURL = () => apiConfig.matchingService.baseURL; export const getQuestionServiceURL = () => apiConfig.questionService.baseURL; -export const getCollaborationHTTPURL = () => apiConfig.collaborationService.httpURL; -export const getCollaborationWSURL = () => apiConfig.collaborationService.wsURL; +export const getCollaborationURL = () => apiConfig.collaborationService.baseURL; diff --git a/frontend/src/stores/collaboration-store.ts b/frontend/src/stores/collaboration-store.ts index 9ccb23a4c0..9db9a3b39c 100644 --- a/frontend/src/stores/collaboration-store.ts +++ b/frontend/src/stores/collaboration-store.ts @@ -3,8 +3,8 @@ import { subscribeWithSelector } from "zustand/middleware"; import axios, { AxiosError } from "axios"; import { toast } from "react-toastify"; import { ProgrammingLanguage } from "@/utils/enums"; -import { collaborationConfig } from "@/utils/config"; import { collaborationAPI, RoomDetails, questionAPI, Question } from "@/lib/api-client"; +import { getCollaborationURL } from "@/lib/api-config"; let executionTimer: NodeJS.Timeout | null = null; const EXECUTION_TIMEOUT_MS = 60000; // 60s @@ -206,14 +206,11 @@ export const useCollaborationStore = create()( // call POST api try { - const response = await axios.post( - `${collaborationConfig.HTTP_URL}/api/v1/code/submit-code`, - { - room_id: roomDetails.roomId, - language: roomDetails.programmingLanguage, - source_code: sourceCode, - }, - ); + const response = await axios.post(`${getCollaborationURL()}/api/v1/code/submit-code`, { + room_id: roomDetails.roomId, + language: roomDetails.programmingLanguage, + source_code: sourceCode, + }); if (response.status !== 202) { throw new Error(response.data.error || "Failed to submit code"); diff --git a/frontend/src/utils/config.ts b/frontend/src/utils/config.ts deleted file mode 100644 index 8ddbeda485..0000000000 --- a/frontend/src/utils/config.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Stores configuration constants for frontend to interact with the backend services. - * @deprecated Use apiConfig from @/lib/api-config instead - */ - -import { apiConfig } from "@/lib/api-config"; - -export const collaborationConfig = { - HTTP_URL: apiConfig.collaborationService.httpURL, - WS_URL: apiConfig.collaborationService.wsURL, -}; From 6e81071d1b035fc29cc7e6b714435f963068aabe Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 07:59:24 +0800 Subject: [PATCH 02/12] refactor(user-service): hide EMAIL_USER env, remove unused API_BASE_URL env --- backend/user-service/.env.example | 1 + docker-compose.yml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/user-service/.env.example b/backend/user-service/.env.example index 031a97a734..6074923241 100644 --- a/backend/user-service/.env.example +++ b/backend/user-service/.env.example @@ -1,2 +1,3 @@ +EMAIL_USER="" EMAIL_PASSWORD="" JWT_SECRET="" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8eab1de753..bb2b5c6c40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -125,9 +125,7 @@ services: environment: MONGO_URI: mongodb://admin:password@mongodb:27017/peerprepUserDB?authSource=admin WEB_BASE_URL: http://localhost:3000 - API_BASE_URL: http://localhost:8001 PORT: 8001 - EMAIL_USER: peerprep.team@gmail.com networks: - cs3219-peerprep-network From 59292e34ea8e501077b98d427bdad1003e1a8136 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 10:28:56 +0800 Subject: [PATCH 03/12] refactor(question-service): remove hardcode fronend URL --- backend/question-service/src/index.js | 12 +++++++----- docker-compose.yml | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/question-service/src/index.js b/backend/question-service/src/index.js index 8b1ddea8cd..79759487e8 100644 --- a/backend/question-service/src/index.js +++ b/backend/question-service/src/index.js @@ -8,11 +8,13 @@ connectDB() const app = express() -app.use(cors({ - origin: 'http://localhost:3000', - credentials: true, - optionsSuccessStatus: 200, -})); +app.use( + cors({ + origin: process.env.WEB_BASE_URL, + credentials: true, + optionsSuccessStatus: 200, + }), +); app.use(express.json()) app.use(express.urlencoded({ extended: false })) diff --git a/docker-compose.yml b/docker-compose.yml index bb2b5c6c40..d5b5ff21c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -217,6 +217,7 @@ services: - '8003:8003' environment: - MONGO_URI=mongodb://admin:password@mongodb:27017/peerprepQuestionServiceDB?authSource=admin + - WEB_BASE_URL=http://localhost:3000 - PORT=8003 - KAFKA_HOST=kafka - KAFKA_PORT=9092 From abb503fe60988a00e1bb1feab24feabf6268b258 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 13:57:28 +0800 Subject: [PATCH 04/12] refactor(frontend): add docker build arguments --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d5b5ff21c1..68a6abb3b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,15 +234,15 @@ services: build: context: ./frontend dockerfile: Dockerfile.frontend + args: + - USER_SERVICE_URL=http://localhost:8001 + - MATCHING_SERVICE_URL=http://localhost:8002 + - QUESTION_SERVICE_URL=http://localhost:8003 + - COLLABORATION_SERVICE_URL=http://localhost:8004 container_name: frontend restart: always ports: - '3000:3000' - environment: - - NEXT_PUBLIC_USER_SERVICE_URL=http://localhost:8001 - - NEXT_PUBLIC_MATCHING_SERVICE_URL=http://localhost:8002 - - NEXT_PUBLIC_QUESTION_SERVICE_URL=http://localhost:8003 - - NEXT_PUBLIC_COLLABORATION_SERVICE_URL=http://localhost:8004 depends_on: - user-service - collaboration-service From 4d9603fcd476b7b2a4c8b66fb40e3f67de544d3b Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 14:48:13 +0800 Subject: [PATCH 05/12] fix(matching-service): get url from correct environment variable --- frontend/src/stores/matching-store.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frontend/src/stores/matching-store.ts b/frontend/src/stores/matching-store.ts index 09f7381206..87dbc955ad 100644 --- a/frontend/src/stores/matching-store.ts +++ b/frontend/src/stores/matching-store.ts @@ -4,6 +4,7 @@ import { Socket } from "socket.io-client"; import { toast } from "react-toastify"; import { ServiceType } from "@/utils/enums"; import { socketManager } from "@/utils/socket-manager"; +import { getMatchingServiceURL } from "@/lib/api-config"; interface User { _id: string; @@ -68,10 +69,7 @@ export const useMatchingStore = create()( return; } - const url = - process.env.NEXT_PUBLIC_ENV === "production" - ? process.env.NEXT_PUBLIC_MATCHING_ENDPOINT - : `http://localhost:8002`; + const url = getMatchingServiceURL(); // Get access token for authentication const accessToken = localStorage.getItem("accessToken"); From e6294c8b6cf3c124b2d1f8a11ddb8917902dea63 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 16:21:24 +0800 Subject: [PATCH 06/12] refactor(matching-service): modify socket url extraction --- frontend/src/stores/matching-store.ts | 10 ++++++++-- frontend/src/utils/socket-manager.ts | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frontend/src/stores/matching-store.ts b/frontend/src/stores/matching-store.ts index 87dbc955ad..9c5a8715cf 100644 --- a/frontend/src/stores/matching-store.ts +++ b/frontend/src/stores/matching-store.ts @@ -70,14 +70,20 @@ export const useMatchingStore = create()( } const url = getMatchingServiceURL(); + const urlObj = new URL(url); + const domainUrl = `${urlObj.protocol}//${urlObj.host}`; + const baseSocketPath = urlObj.pathname === "/" ? "" : urlObj.pathname; + const socketPath = `${baseSocketPath}/socket.io`; // Get access token for authentication const accessToken = localStorage.getItem("accessToken"); // Create socket using socketManager const newSocket = socketManager.createSocket(ServiceType.MATCHING, { - url: url || "", - options: {}, + url: domainUrl, + options: { + path: socketPath, + }, token: accessToken || undefined, }); diff --git a/frontend/src/utils/socket-manager.ts b/frontend/src/utils/socket-manager.ts index bc04540094..8194c8a90c 100644 --- a/frontend/src/utils/socket-manager.ts +++ b/frontend/src/utils/socket-manager.ts @@ -1,8 +1,8 @@ -import { io, Socket, SocketOptions } from "socket.io-client"; +import { io, Socket, ManagerOptions, SocketOptions } from "socket.io-client"; interface SocketConfig { url: string; - options?: Partial; + options?: Partial; token?: string; } From 57eb2d16c4fbe6a070676b7cbb3a50b6bcb032df Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 16:23:44 +0800 Subject: [PATCH 07/12] feat(kafka): add gcp pubsub as alternative --- backend/collaboration-service/package.json | 1 + .../collaboration-service/src/config/kafka.js | 8 + .../src/config/pubsub.js | 179 +++++ .../src/consumers/roomCreationConsumer.js | 22 +- backend/execution-service/src/worker.js | 1 + backend/matching-service/package.json | 1 + backend/matching-service/src/config/kafka.ts | 10 + backend/matching-service/src/config/pubsub.ts | 237 ++++++ backend/matching-service/src/index.ts | 30 +- .../src/workers/matchingWorker.ts | 11 +- backend/question-service/package.json | 1 + backend/question-service/src/config/kafka.js | 8 + backend/question-service/src/config/pubsub.js | 203 +++++ .../src/controllers/questionController.js | 6 +- backend/question-service/src/index.js | 9 +- pnpm-lock.yaml | 720 +++++++++++++++++- 16 files changed, 1423 insertions(+), 24 deletions(-) create mode 100644 backend/collaboration-service/src/config/pubsub.js create mode 100644 backend/matching-service/src/config/pubsub.ts create mode 100644 backend/question-service/src/config/pubsub.js diff --git a/backend/collaboration-service/package.json b/backend/collaboration-service/package.json index 7a7c95eb35..33d300f9b2 100644 --- a/backend/collaboration-service/package.json +++ b/backend/collaboration-service/package.json @@ -14,6 +14,7 @@ "license": "ISC", "packageManager": "pnpm@10.17.1", "dependencies": { + "@google-cloud/pubsub": "^5.2.0", "@y/websocket-server": "^0.1.1", "axios": "^1.7.9", "amqplib": "^0.10.9", diff --git a/backend/collaboration-service/src/config/kafka.js b/backend/collaboration-service/src/config/kafka.js index 3fdccc0a7d..8c9a13ee3d 100644 --- a/backend/collaboration-service/src/config/kafka.js +++ b/backend/collaboration-service/src/config/kafka.js @@ -7,6 +7,14 @@ export const ROOM_CREATED_TOPIC = 'room_created_topic'; export class KafkaManager { constructor() { console.log('Collaboration Service KafkaManager constructor'); + const isGCP = !!process.env.PUBSUB_PROJECT_ID; + if (isGCP) { + console.log('GCP environment detected, Kafka not configured for GCP'); + this.kafka = null; + this.producer = null; + this.consumer = null; + return; + } const host = process.env.KAFKA_HOST || 'localhost'; const port = process.env.KAFKA_PORT || '29092'; const brokers = (process.env.KAFKA_BROKERS || `${host}:${port}`).split(','); diff --git a/backend/collaboration-service/src/config/pubsub.js b/backend/collaboration-service/src/config/pubsub.js new file mode 100644 index 0000000000..0427c4b891 --- /dev/null +++ b/backend/collaboration-service/src/config/pubsub.js @@ -0,0 +1,179 @@ +import { PubSub } from "@google-cloud/pubsub"; + +export const ROOM_CREATION_TOPIC = "room_creation_topic"; +export const ROOM_CREATED_TOPIC = "room_created_topic"; + +export const ROOM_CREATION_SUBSCRIPTION = "room_creation_sub"; + +export class PubSubManager { + constructor() { + console.log("Collaboration Service PubSubManager constructor"); + + // Check if running on GCP (PUBSUB_PROJECT_ID is set) + this.useGcp = !!process.env.PUBSUB_PROJECT_ID; + + if (this.useGcp) { + console.log("Using GCP Pub/Sub"); + this.pubsub = new PubSub({ + projectId: process.env.PUBSUB_PROJECT_ID, + }); + } else { + console.log("GCP Pub/Sub not configured, using fallback mode"); + this.pubsub = null; + } + + this.subscriptions = new Map(); + this.isConnected = false; + } + + async initWithRetry(maxRetries = 5, retryDelayMs = 2000) { + if (this.isConnected) { + console.log("Already connected to Pub/Sub"); + return; + } + + if (!this.useGcp) { + console.log("Skipping Pub/Sub initialization (not on GCP)"); + this.isConnected = true; + return; + } + + let attempt = 0; + while (attempt < maxRetries) { + try { + // Verify connection by listing topics + const [topics] = await this.pubsub.getTopics(); + console.log(`Connected to Pub/Sub. Found ${topics.length} topics`); + + // Ensure topics exist + await this.ensureTopicsExist([ROOM_CREATION_TOPIC, ROOM_CREATED_TOPIC]); + + this.isConnected = true; + console.log("Connected to Pub/Sub"); + return; + } catch (err) { + attempt += 1; + console.error(`Failed to connect to Pub/Sub (attempt ${attempt} of ${maxRetries}):`, err); + if (attempt >= maxRetries) { + throw err; + } + await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt)); + } + } + } + + async ensureTopicsExist(topicNames) { + for (const topicName of topicNames) { + try { + const topic = this.pubsub.topic(topicName); + const [exists] = await topic.exists(); + + if (!exists) { + await topic.create(); + console.log(`Created topic: ${topicName}`); + } + } catch (error) { + console.error(`Error ensuring topic ${topicName}:`, error); + } + } + } + + async setupConsumer(handler) { + await this.initWithRetry(); + + if (!this.useGcp) { + console.log("Skipping Pub/Sub consumer setup (not on GCP)"); + return; + } + + await this.setupSubscription(ROOM_CREATION_TOPIC, ROOM_CREATION_SUBSCRIPTION, handler); + } + + async setupSubscription(topicName, subscriptionName, handler) { + try { + const topic = this.pubsub.topic(topicName); + let subscription = topic.subscription(subscriptionName); + + const [exists] = await subscription.exists(); + if (!exists) { + [subscription] = await topic.createSubscription(subscriptionName); + console.log(`Created subscription: ${subscriptionName}`); + } + + // Handle messages + const messageHandler = async (message) => { + try { + const value = message.data.toString(); + const key = message.attributes.key || null; + + await handler({ + key, + value, + }); + + message.ack(); + } catch (error) { + console.error(`Error handling message from ${topicName}:`, error); + message.nack(); + } + }; + + subscription.on("message", messageHandler); + subscription.on("error", (error) => { + console.error(`Subscription ${subscriptionName} error:`, error); + }); + + this.subscriptions.set(subscriptionName, subscription); + console.log(`Subscribed to ${topicName} (${subscriptionName})`); + } catch (error) { + console.error(`Error setting up subscription for ${topicName}:`, error); + throw error; + } + } + + async publishRoomCreated(matchId, roomId) { + if (!this.useGcp) { + console.log(`[Fallback] Would publish room created: ${matchId} -> ${roomId}`); + return; + } + + try { + const topic = this.pubsub.topic(ROOM_CREATED_TOPIC); + const data = { roomId }; + const dataBuffer = Buffer.from(JSON.stringify(data)); + + await topic.publishMessage({ + data: dataBuffer, + attributes: { key: matchId }, + }); + + console.log("Published room created event:", { matchId, roomId }); + } catch (error) { + console.error("Error publishing room created event:", error); + throw error; + } + } + + async disconnect() { + if (!this.useGcp) { + return; + } + + try { + for (const [name, subscription] of this.subscriptions) { + await subscription.close(); + console.log(`Closed subscription: ${name}`); + } + this.subscriptions.clear(); + + if (this.pubsub) { + await this.pubsub.close(); + } + console.log("Disconnected from Pub/Sub"); + } catch (error) { + console.error("Error disconnecting from Pub/Sub:", error); + } + } +} + +export const pubsubManager = new PubSubManager(); diff --git a/backend/collaboration-service/src/consumers/roomCreationConsumer.js b/backend/collaboration-service/src/consumers/roomCreationConsumer.js index 0594dae2e7..33c575dfcb 100644 --- a/backend/collaboration-service/src/consumers/roomCreationConsumer.js +++ b/backend/collaboration-service/src/consumers/roomCreationConsumer.js @@ -1,5 +1,8 @@ import roomController from '../controllers/roomController.js'; import { kafkaManager } from '../config/kafka.js'; +import { pubsubManager } from '../config/pubsub.js'; + +const useGcp = !!process.env.PUBSUB_PROJECT_ID; export async function handleRoomCreation(message) { const matchId = message.key; @@ -26,16 +29,25 @@ export async function handleRoomCreation(message) { console.log('Room created successfully:', { matchId, roomId }); - await kafkaManager.publishRoomCreated(matchId, roomId); - + if (useGcp) { + await pubsubManager.publishRoomCreated(matchId, roomId); + } else { + await kafkaManager.publishRoomCreated(matchId, roomId); + } } catch (error) { console.error('Error handling room creation request:', error) } } export async function setupRoomCreationConsumer() { - console.log('Setting up room creation Kafka consumer...'); - await kafkaManager.setupConsumer(handleRoomCreation); - console.log('Room creation Kafka consumer started'); + if (useGcp) { + console.log('Setting up room creation Pub/Sub consumer...'); + await pubsubManager.setupConsumer(handleRoomCreation); + console.log('Room creation Pub/Sub consumer started'); + } else { + console.log('Setting up room creation Kafka consumer...'); + await kafkaManager.setupConsumer(handleRoomCreation); + console.log('Room creation Kafka consumer started'); + } } diff --git a/backend/execution-service/src/worker.js b/backend/execution-service/src/worker.js index b9329960b1..72c7b14376 100644 --- a/backend/execution-service/src/worker.js +++ b/backend/execution-service/src/worker.js @@ -15,6 +15,7 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); async function callPistonAPI(language, source_code, timeout_ms) { try { + const pistonURL = "http://piston:2000/api/v2/execute" const payload = { language: language, version: "*", diff --git a/backend/matching-service/package.json b/backend/matching-service/package.json index 107f219668..eb2c01f8c6 100644 --- a/backend/matching-service/package.json +++ b/backend/matching-service/package.json @@ -14,6 +14,7 @@ "license": "ISC", "packageManager": "pnpm@10.17.1", "dependencies": { + "@google-cloud/pubsub": "^5.2.0", "@types/jsonwebtoken": "^9.0.10", "cors": "^2.8.5", "dotenv": "^17.2.3", diff --git a/backend/matching-service/src/config/kafka.ts b/backend/matching-service/src/config/kafka.ts index b72cdd39ee..7409f53417 100644 --- a/backend/matching-service/src/config/kafka.ts +++ b/backend/matching-service/src/config/kafka.ts @@ -20,6 +20,16 @@ export class KafkaManager { constructor() { console.log('KafkaManager constructor'); + const isGCP = !!process.env.PUBSUB_PROJECT_ID; + if (isGCP) { + console.log('GCP environment detected, Kafka not configured for GCP'); + this.kafka = null as unknown as Kafka; + this.admin = null as unknown as Admin; + this.producer = null as unknown as Producer; + this.consumer_of_question_topic = null as unknown as Consumer; + this.consumer_of_room_created_topic = null as unknown as Consumer; + return; + } const host = process.env.KAFKA_HOST || 'localhost'; const port = process.env.KAFKA_PORT || '9092'; const brokers = (process.env.KAFKA_BROKERS || `${host}:${port}`).split(','); diff --git a/backend/matching-service/src/config/pubsub.ts b/backend/matching-service/src/config/pubsub.ts new file mode 100644 index 0000000000..df1fcf717f --- /dev/null +++ b/backend/matching-service/src/config/pubsub.ts @@ -0,0 +1,237 @@ +import { PubSub, Message, Subscription } from "@google-cloud/pubsub"; + +export const MATCH_TOPIC = "match_topic"; +export const QUESTION_TOPIC = "question_topic"; +export const ROOM_CREATION_TOPIC = "room_creation_topic"; +export const ROOM_CREATED_TOPIC = "room_created_topic"; + +export const QUESTION_SUBSCRIPTION = "question_sub"; +export const ROOM_CREATED_SUBSCRIPTION = "room_created_sub"; + +type MessageHandler = (message: { + key?: string | null; + value?: string | null; +}) => Promise | void; + +export class PubSubManager { + private pubsub: PubSub; + private isConnected = false; + private subscriptions: Map = new Map(); + private useGcp: boolean; + + constructor() { + console.log("PubSubManager constructor"); + + // Check if running on GCP (PUBSUB_PROJECT_ID is set) + this.useGcp = !!process.env.PUBSUB_PROJECT_ID; + + if (this.useGcp) { + console.log("Using GCP Pub/Sub"); + this.pubsub = new PubSub({ + projectId: process.env.PUBSUB_PROJECT_ID, + }); + } else { + console.log("GCP Pub/Sub not configured, using fallback mode"); + // In development without GCP, we'll create a stub + this.pubsub = null as any; + } + } + + async initWithRetry(maxRetries = 5, retryDelayMs = 2000): Promise { + if (this.isConnected) { + console.log("Already connected to Pub/Sub"); + return; + } + + if (!this.useGcp) { + console.log("Skipping Pub/Sub initialization (not on GCP)"); + this.isConnected = true; + return; + } + + let attempt = 0; + while (attempt < maxRetries) { + try { + // Verify connection by listing topics + const [topics] = await this.pubsub.getTopics(); + console.log(`Connected to Pub/Sub. Found ${topics.length} topics`); + + // Ensure topics exist + await this.ensureTopicsExist([ + MATCH_TOPIC, + QUESTION_TOPIC, + ROOM_CREATION_TOPIC, + ROOM_CREATED_TOPIC, + ]); + + this.isConnected = true; + console.log("Connected to Pub/Sub"); + return; + } catch (err) { + attempt += 1; + console.error(`Failed to connect to Pub/Sub (attempt ${attempt} of ${maxRetries}):`, err); + if (attempt >= maxRetries) { + throw err; + } + await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt)); + } + } + } + + private async ensureTopicsExist(topicNames: string[]): Promise { + for (const topicName of topicNames) { + try { + const topic = this.pubsub.topic(topicName); + const [exists] = await topic.exists(); + + if (!exists) { + await topic.create(); + console.log(`Created topic: ${topicName}`); + } + } catch (error) { + console.error(`Error ensuring topic ${topicName}:`, error); + } + } + } + + async publishMessage( + topicName: string, + data: any, + attributes?: Record, + ): Promise { + if (!this.useGcp) { + console.log(`[Fallback] Would publish to ${topicName}:`, data); + return; + } + + try { + const topic = this.pubsub.topic(topicName); + const dataBuffer = Buffer.from(JSON.stringify(data)); + + await topic.publishMessage({ + data: dataBuffer, + attributes: attributes || {}, + }); + + console.log(`Published message to ${topicName}`); + } catch (error) { + console.error(`Error publishing to ${topicName}:`, error); + throw error; + } + } + + async setupSubscribers(handlers: { + onCollabMessage?: MessageHandler; + onQuestionMessage?: MessageHandler; + onRoomCreatedMessage?: MessageHandler; + }): Promise { + await this.initWithRetry(); + + if (!this.useGcp) { + console.log("Skipping Pub/Sub subscribers setup (not on GCP)"); + return; + } + + // Setup question topic subscription + if (handlers.onQuestionMessage) { + await this.setupSubscription(QUESTION_TOPIC, QUESTION_SUBSCRIPTION, handlers.onQuestionMessage); + } + + // Setup room created topic subscription + if (handlers.onRoomCreatedMessage) { + await this.setupSubscription( + ROOM_CREATED_TOPIC, + ROOM_CREATED_SUBSCRIPTION, + handlers.onRoomCreatedMessage, + ); + } + } + + private async setupSubscription( + topicName: string, + subscriptionName: string, + handler: MessageHandler, + ): Promise { + try { + const topic = this.pubsub.topic(topicName); + let subscription = topic.subscription(subscriptionName); + + const [exists] = await subscription.exists(); + if (!exists) { + [subscription] = await topic.createSubscription(subscriptionName); + console.log(`Created subscription: ${subscriptionName}`); + } + + // Handle messages + const messageHandler = async (message: Message) => { + try { + const value = message.data.toString(); + const key = message.attributes.key || null; + + await handler({ key, value }); + message.ack(); + } catch (error) { + console.error(`Error handling message from ${topicName}:`, error); + message.nack(); + } + }; + + subscription.on("message", messageHandler); + subscription.on("error", (error) => { + console.error(`Subscription ${subscriptionName} error:`, error); + }); + + this.subscriptions.set(subscriptionName, subscription); + console.log(`Subscribed to ${topicName} (${subscriptionName})`); + } catch (error) { + console.error(`Error setting up subscription for ${topicName}:`, error); + throw error; + } + } + + async clearAllTopics(): Promise { + // Not applicable for Pub/Sub - topics are managed externally + console.log("clearAllTopics: Not applicable for Pub/Sub"); + } + + async resetConsumerOffsets(): Promise { + // Not applicable for Pub/Sub - no concept of offsets + console.log("resetConsumerOffsets: Not applicable for Pub/Sub"); + } + + async disconnect(): Promise { + if (!this.useGcp) { + return; + } + + try { + for (const [name, subscription] of this.subscriptions) { + await subscription.close(); + console.log(`Closed subscription: ${name}`); + } + this.subscriptions.clear(); + + await this.pubsub.close(); + console.log("Disconnected from Pub/Sub"); + } catch (error) { + console.error("Error disconnecting from Pub/Sub:", error); + } + } + + // Legacy method name compatibility + getProducer(): any { + return { + send: async ({ topic, messages }: any) => { + if (!messages || messages.length === 0) return; + + const message = messages[0]; + const data = message.value ? JSON.parse(message.value) : {}; + const attributes = message.key ? { key: message.key } : undefined; + + await this.publishMessage(topic, data, attributes); + }, + }; + } +} + +export const pubsubManager = new PubSubManager(); diff --git a/backend/matching-service/src/index.ts b/backend/matching-service/src/index.ts index edd617f431..9cae872c3d 100644 --- a/backend/matching-service/src/index.ts +++ b/backend/matching-service/src/index.ts @@ -10,6 +10,7 @@ import { redisConfig } from './config/redis'; import { initSocket } from './config/socket'; import { startMatchingWorker } from './workers/matchingWorker'; import { kafkaManager } from './config/kafka'; +import { pubsubManager } from './config/pubsub'; import { handleQuestionMessage, handleRoomCreatedMessage } from './workers/matchingWorker'; import { matchingController, cleanupConsumerGroup } from './controllers/matchingController'; @@ -22,6 +23,8 @@ app.get('/health', (_req: any, res: any) => { res.json({ status: 'OK', service: 'matching-service' }); }); +const useGcp = !!process.env.PUBSUB_PROJECT_ID; + async function startServer() { try { const httpServer = createServer(app); @@ -31,11 +34,18 @@ async function startServer() { // Start background matching worker startMatchingWorker(); - // Subscribe to Kafka topics for question services and room creation - await kafkaManager.setupSubscribers({ - onQuestionMessage: handleQuestionMessage, - onRoomCreatedMessage: handleRoomCreatedMessage, - }); + // Subscribe to topics for question services and room creation + if (useGcp) { + await pubsubManager.setupSubscribers({ + onQuestionMessage: handleQuestionMessage, + onRoomCreatedMessage: handleRoomCreatedMessage, + }); + } else { + await kafkaManager.setupSubscribers({ + onQuestionMessage: handleQuestionMessage, + onRoomCreatedMessage: handleRoomCreatedMessage, + }); + } const port = Number(process.env.PORT) || 8002; httpServer.listen(port, () => { @@ -54,11 +64,15 @@ process.on('SIGINT', async () => { await redisConfig.clearAllMatchingData(); await cleanupConsumerGroup(); - await kafkaManager.resetConsumerOffsets(); - await kafkaManager.clearAllTopics(); + if (useGcp) { + await pubsubManager.disconnect(); + } else { + await kafkaManager.resetConsumerOffsets(); + await kafkaManager.clearAllTopics(); + await kafkaManager.disconnect(); + } await redisConfig.disconnect(); - await kafkaManager.disconnect(); console.log('Shutdown completed successfully'); } catch (error) { diff --git a/backend/matching-service/src/workers/matchingWorker.ts b/backend/matching-service/src/workers/matchingWorker.ts index 6942a51703..0fb1641769 100644 --- a/backend/matching-service/src/workers/matchingWorker.ts +++ b/backend/matching-service/src/workers/matchingWorker.ts @@ -1,6 +1,7 @@ import { redis, redisConfig } from '../config/redis'; import { getSocket } from '../config/socket'; import { kafkaManager, MATCH_TOPIC, ROOM_CREATION_TOPIC } from '../config/kafka'; +import { pubsubManager } from '../config/pubsub'; import { User } from '../models/types'; import { v4 as uuidv4 } from 'uuid'; import { SOCKET_EVENTS } from '../constants/socketEvents'; @@ -8,6 +9,8 @@ import { clearMatchCountdownFor } from '../controllers/matchingController'; import { acquireLock, releaseLock } from '../config/redislock'; import { ALL_DIFFICULTIES, ALL_TOPICS, MATCHING_INTERVAL_MS, MATCHING_LOCK_KEY, MATCHING_LOCK_TTL } from '../constants/matchingStatus'; +const useGcp = !!process.env.PUBSUB_PROJECT_ID; + function topicsCompatible(a: string, b: string): boolean { if (!a || !b) return false; return a === ALL_TOPICS || b === ALL_TOPICS || a === b; @@ -44,7 +47,8 @@ async function handleMatch(u1: User, u2: User) { clearMatchCountdownFor(u2.socketId); } - const producer = kafkaManager.getProducer(); + // Use Pub/Sub if on GCP, otherwise use Kafka + const producer = useGcp ? pubsubManager.getProducer() : kafkaManager.getProducer(); console.log(`Match found: ${u1.socketId} + ${u2.socketId} (${u1.topic}/${u1.difficulty})`); const topic = pickFinal(u1.topic, u2.topic); const difficulty = pickFinalDifficulty(u1.difficulty, u2.difficulty); @@ -241,7 +245,8 @@ export async function handleQuestionMessage(message: { key?: string | null; valu userIds: userIds, }; - const producer = kafkaManager.getProducer(); + // Use Pub/Sub if on GCP, otherwise use Kafka + const producer = useGcp ? pubsubManager.getProducer() : kafkaManager.getProducer(); await producer.send({ topic: ROOM_CREATION_TOPIC, messages: [ @@ -252,7 +257,7 @@ export async function handleQuestionMessage(message: { key?: string | null; valu ], }); - console.log('Sent room creation request to Kafka:', roomCreationRequest); + console.log('Sent room creation request:', roomCreationRequest); } catch (error) { console.error('Error sending room creation request to Kafka:', error); diff --git a/backend/question-service/package.json b/backend/question-service/package.json index 77d8d9a1a7..016566701a 100644 --- a/backend/question-service/package.json +++ b/backend/question-service/package.json @@ -12,6 +12,7 @@ "license": "ISC", "packageManager": "pnpm@10.17.1", "dependencies": { + "@google-cloud/pubsub": "^5.2.0", "cloudinary": "^2.8.0", "cors": "^2.8.5", "dotenv": "^17.2.3", diff --git a/backend/question-service/src/config/kafka.js b/backend/question-service/src/config/kafka.js index d395685a27..0c38e05ffa 100644 --- a/backend/question-service/src/config/kafka.js +++ b/backend/question-service/src/config/kafka.js @@ -12,6 +12,14 @@ class KafkaManager { constructor() { console.log('KafkaManager constructor'); + const isGCP = !!process.env.PUBSUB_PROJECT_ID; + if (isGCP) { + console.log('GCP environment detected, Kafka not configured for GCP'); + this.kafka = null; + this.producer = null; + this.consumer = null; + return; + } const host = process.env.KAFKA_HOST || 'localhost'; const port = process.env.KAFKA_PORT || '29092'; const brokers = (process.env.KAFKA_BROKERS || `${host}:${port}`).split(','); diff --git a/backend/question-service/src/config/pubsub.js b/backend/question-service/src/config/pubsub.js new file mode 100644 index 0000000000..1147c33235 --- /dev/null +++ b/backend/question-service/src/config/pubsub.js @@ -0,0 +1,203 @@ +const { PubSub } = require("@google-cloud/pubsub"); +const { getQuestion } = require("../controllers/questionController"); + +const MATCH_TOPIC = "match_topic"; +const QUESTION_TOPIC = "question_topic"; + +const MATCH_SUBSCRIPTION = "match_sub"; + +class PubSubManager { + constructor() { + console.log("PubSubManager constructor"); + + // Check if running on GCP (PUBSUB_PROJECT_ID is set) + this.useGcp = !!process.env.PUBSUB_PROJECT_ID; + + if (this.useGcp) { + console.log("Using GCP Pub/Sub"); + this.pubsub = new PubSub({ + projectId: process.env.PUBSUB_PROJECT_ID, + }); + } else { + console.log("GCP Pub/Sub not configured, using fallback mode"); + this.pubsub = null; + } + + this.subscriptions = new Map(); + this.isConnected = false; + } + + async initWithRetry(maxRetries = 5, retryDelayMs = 2000) { + if (this.isConnected) { + console.log("Already connected to Pub/Sub"); + return; + } + + if (!this.useGcp) { + console.log("Skipping Pub/Sub initialization (not on GCP)"); + this.isConnected = true; + return; + } + + let attempt = 0; + while (attempt < maxRetries) { + try { + // Verify connection by listing topics + const [topics] = await this.pubsub.getTopics(); + console.log(`Connected to Pub/Sub. Found ${topics.length} topics`); + + // Ensure topics exist + await this.ensureTopicsExist([MATCH_TOPIC, QUESTION_TOPIC]); + + this.isConnected = true; + console.log("Connected to Pub/Sub"); + return; + } catch (err) { + attempt += 1; + console.error(`Failed to connect to Pub/Sub (attempt ${attempt} of ${maxRetries}):`, err); + if (attempt >= maxRetries) { + throw err; + } + await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt)); + } + } + } + + async ensureTopicsExist(topicNames) { + for (const topicName of topicNames) { + try { + const topic = this.pubsub.topic(topicName); + const [exists] = await topic.exists(); + + if (!exists) { + await topic.create(); + } + } catch (error) { + console.error(`Error ensuring topic ${topicName}:`, error); + } + } + } + + async publishMessage(topicName, data, attributes = {}) { + if (!this.useGcp) { + console.log(`[Fallback] Would publish to ${topicName}:`, data); + return; + } + + try { + const topic = this.pubsub.topic(topicName); + const dataBuffer = Buffer.from(JSON.stringify(data)); + + await topic.publishMessage({ + data: dataBuffer, + attributes, + }); + + console.log(`Published message to ${topicName}`); + } catch (error) { + console.error(`Error publishing to ${topicName}:`, error); + throw error; + } + } + + getProducer() { + return { + send: async ({ topic, messages }) => { + if (!messages || messages.length === 0) return; + + const message = messages[0]; + const data = message.value ? JSON.parse(message.value) : {}; + const attributes = message.key ? { key: message.key } : {}; + + await this.publishMessage(topic, data, attributes); + }, + }; + } + + async setupSubscribers() { + await this.initWithRetry(); + + if (!this.useGcp) { + console.log("Skipping Pub/Sub subscribers setup (not on GCP)"); + return; + } + + await this.setupSubscription(MATCH_TOPIC, MATCH_SUBSCRIPTION, async (message) => { + await getQuestion(message, this, QUESTION_TOPIC); + }); + } + + async setupSubscription(topicName, subscriptionName, handler) { + try { + const topic = this.pubsub.topic(topicName); + let subscription = topic.subscription(subscriptionName); + + const [exists] = await subscription.exists(); + if (!exists) { + [subscription] = await topic.createSubscription(subscriptionName); + console.log(`Created subscription: ${subscriptionName}`); + } + + // Handle messages + const messageHandler = async (message) => { + try { + const value = message.data.toString(); + const key = message.attributes.key || null; + + // Create message object compatible with Kafka format + const kafkaCompatibleMessage = { + key: key ? Buffer.from(key) : null, + value: Buffer.from(value), + }; + + await handler(kafkaCompatibleMessage); + message.ack(); + } catch (error) { + console.error(`Error handling message from ${topicName}:`, error); + message.nack(); + } + }; + + subscription.on("message", messageHandler); + subscription.on("error", (error) => { + console.error(`Subscription ${subscriptionName} error:`, error); + }); + + this.subscriptions.set(subscriptionName, subscription); + console.log(`Subscribed to ${topicName} (${subscriptionName})`); + } catch (error) { + console.error(`Error setting up subscription for ${topicName}:`, error); + throw error; + } + } + + async disconnect() { + if (!this.useGcp) { + return; + } + + try { + for (const [name, subscription] of this.subscriptions) { + await subscription.close(); + console.log(`Closed subscription: ${name}`); + } + this.subscriptions.clear(); + + if (this.pubsub) { + await this.pubsub.close(); + } + console.log("Disconnected from Pub/Sub"); + } catch (error) { + console.error("Error disconnecting from Pub/Sub:", error); + } + } +} + +const pubsubManager = new PubSubManager(); + +module.exports = { + PubSubManager, + pubsubManager, + MATCH_TOPIC, + QUESTION_TOPIC, +}; diff --git a/backend/question-service/src/controllers/questionController.js b/backend/question-service/src/controllers/questionController.js index a025bb0cf3..6852343c1c 100644 --- a/backend/question-service/src/controllers/questionController.js +++ b/backend/question-service/src/controllers/questionController.js @@ -407,11 +407,11 @@ exports.uploadQuestionImage = async (req, res) => { * @param {Object} message - Kafka message containing matching criteria * @param {string} message.key - Message key (optional) * @param {string} message.value - JSON string containing topic and difficulty - * @param {Object} kafkaManager - Kafka manager instance for sending responses + * @param {Object} messageManager - Message manager instance (Kafka or Pub/Sub) to send response * @param {string} questionTopic - Topic to send the question response to * @returns {Object} Question object or error response */ -const getQuestion = async (message, kafkaManager, questionTopic) => { +const getQuestion = async (message, messageManager, questionTopic) => { try { if (!message.value) { console.error('No message value provided'); @@ -468,7 +468,7 @@ const getQuestion = async (message, kafkaManager, questionTopic) => { }); const matchId = message.key?.toString(); - const producer = kafkaManager.getProducer(); + const producer = messageManager.getProducer(); await producer.send({ topic: questionTopic, messages: [ diff --git a/backend/question-service/src/index.js b/backend/question-service/src/index.js index 79759487e8..de02eb67ee 100644 --- a/backend/question-service/src/index.js +++ b/backend/question-service/src/index.js @@ -3,6 +3,7 @@ const cors = require('cors') const dotenv = require('dotenv').config() const connectDB = require('./config/db') const { kafkaManager } = require('./config/kafka') +const { pubsubManager } = require('./config/pubsub') connectDB() @@ -20,10 +21,16 @@ app.use(express.json()) app.use(express.urlencoded({ extended: false })) const PORT = process.env.PORT || 8003 +const useGcp = !!process.env.PUBSUB_PROJECT_ID; app.listen(PORT, async () => { console.log(`Question service is running on port ${PORT}...`) - await kafkaManager.setupSubscribers() + + if (useGcp) { + await pubsubManager.setupSubscribers(); + } else { + await kafkaManager.setupSubscribers(); + } }) app.get('/', (req, res) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37d52ffd9b..a12c4f182f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: backend/collaboration-service: dependencies: + '@google-cloud/pubsub': + specifier: ^5.2.0 + version: 5.2.0 '@y/websocket-server': specifier: ^0.1.1 version: 0.1.1(yjs@13.6.27) @@ -88,6 +91,9 @@ importers: backend/matching-service: dependencies: + '@google-cloud/pubsub': + specifier: ^5.2.0 + version: 5.2.0 '@types/jsonwebtoken': specifier: ^9.0.10 version: 9.0.10 @@ -146,6 +152,9 @@ importers: backend/question-service: dependencies: + '@google-cloud/pubsub': + specifier: ^5.2.0 + version: 5.2.0 cloudinary: specifier: ^2.8.0 version: 2.8.0 @@ -349,7 +358,7 @@ importers: version: 12.23.24(@emotion/is-prop-valid@1.4.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) next: specifier: 15.5.4 - version: 15.5.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + version: 15.5.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) node-domexception: specifier: ^2.0.2 version: 2.0.2 @@ -673,6 +682,35 @@ packages: '@floating-ui/utils@0.2.10': resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==} + '@google-cloud/paginator@6.0.0': + resolution: {integrity: sha512-g5nmMnzC+94kBxOKkLGpK1ikvolTFCC3s2qtE4F+1EuArcJ7HHC23RDQVt3Ra3CqpUYZ+oXNKZ8n5Cn5yug8DA==} + engines: {node: '>=18'} + + '@google-cloud/precise-date@5.0.0': + resolution: {integrity: sha512-9h0Gvw92EvPdE8AK8AgZPbMnH5ftDyPtKm7/KUfcJVaPEPjwGDsJd1QV0H8esBDV4II41R/2lDWH1epBqIoKUw==} + engines: {node: '>=18'} + + '@google-cloud/projectify@5.0.0': + resolution: {integrity: sha512-XXQLaIcLrOAMWvRrzz+mlUGtN6vlVNja3XQbMqRi/V7XJTAVwib3VcKd7oRwyZPkp7rBVlHGcaqdyGRrcnkhlA==} + engines: {node: '>=18'} + + '@google-cloud/promisify@5.0.0': + resolution: {integrity: sha512-N8qS6dlORGHwk7WjGXKOSsLjIjNINCPicsOX6gyyLiYk7mq3MtII96NZ9N2ahwA2vnkLmZODOIH9rlNniYWvCQ==} + engines: {node: '>=18'} + + '@google-cloud/pubsub@5.2.0': + resolution: {integrity: sha512-YNSRBo85mgPQ9QuuzAHjmLwngIwmy2RjAUAoPl2mOL2+bCM0cAVZswPb8ylcsWJP7PgDJlck+ybv0MwJ9AM0sg==} + engines: {node: '>=18'} + + '@grpc/grpc-js@1.14.1': + resolution: {integrity: sha512-sPxgEWtPUR3EnRJCEtbGZG2iX8LQDUls2wUS3o27jg07KqJFMq6YDeWvMo1wfpmy3rqRdS0rivpLwhqQtEyCuQ==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.8.0': + resolution: {integrity: sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==} + engines: {node: '>=6'} + hasBin: true + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -819,6 +857,10 @@ packages: cpu: [x64] os: [win32] + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} @@ -842,6 +884,9 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@monaco-editor/loader@1.6.1': resolution: {integrity: sha512-w3tEnj9HYEC73wtjdpR089AqkUPskFRcdkxsiSFt3SoUc3OHpmu+leP94CXBm4mHfefmhsdfI0ZQu6qJ0wgtPg==} @@ -1022,6 +1067,28 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + + '@opentelemetry/semantic-conventions@1.34.0': + resolution: {integrity: sha512-aKcOkyrorBGlajjRdVoJWHTxfxO1vCNHLJVlSDaRHDIdjU+pX8IYQPvPDkYiujKLbRnWU+1TBwEt0QRgSm4SGA==} + engines: {node: '>=14'} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@pkgr/core@0.2.9': resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -1029,6 +1096,36 @@ packages: '@popperjs/core@2.11.8': resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@radix-ui/number@1.1.1': resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} @@ -1593,6 +1690,10 @@ packages: resolution: {integrity: sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg==} engines: {node: '>=12'} + '@tootallnate/once@2.0.0': + resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} + engines: {node: '>= 10'} + '@tsconfig/node10@1.0.11': resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} @@ -1902,6 +2003,14 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + agora-access-token@2.0.4: resolution: {integrity: sha512-RtOIvi4PqV1ok3rdnopMZeVwiUqZgG9Pp56kqxJc5cU/+ljRBzs1Al+IThD5ahm528dGsdY1TyP1bJdqmgBCbQ==} deprecated: The package has been deprecated. Please use 'agora-token' instead, sorry for the inconvenience caused. @@ -1923,6 +2032,10 @@ packages: resolution: {integrity: sha512-Zhl0ErHcSRUaVfGUeUdDuLgpkEo8KIFjB4Y9uAc46ScOpdDiU1Dbyplh7qWJeJ/ZHpbyMSM26+X3BySgnIz40Q==} engines: {node: '>=18'} + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + ansi-regex@6.2.2: resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} engines: {node: '>=12'} @@ -1991,6 +2104,10 @@ packages: resolution: {integrity: sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==} engines: {node: '>= 0.4'} + arrify@2.0.1: + resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} + engines: {node: '>=8'} + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -2037,6 +2154,9 @@ packages: resolution: {integrity: sha512-cU8v/EGSrnH+HnxV2z0J7/blxH8gq7Xh2JFT6Aroax7UohdmiJJlxApMxtKfuI7z68NvvVcmR78k2LbT6efhRg==} engines: {node: '>= 18'} + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -2138,6 +2258,10 @@ packages: client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + cloudinary@2.8.0: resolution: {integrity: sha512-s7frvR0HnQXeJsQSIsbLa/I09IMb1lOnVLEDH5b5E53WTiCYgrNNOBGV/i/nLHwrcEOUkqjfSwP1+enXWNYmdw==} engines: {node: '>=9'} @@ -2248,6 +2372,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-view-buffer@1.0.2: resolution: {integrity: sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==} engines: {node: '>= 0.4'} @@ -2361,9 +2489,15 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + duplexify@4.1.3: + resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==} + dynamic-dedupe@0.3.0: resolution: {integrity: sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==} + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} @@ -2386,6 +2520,9 @@ packages: emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} @@ -2402,6 +2539,9 @@ packages: engines: {node: '>=6'} deprecated: Superseded by abstract-level (https://github.com/Level/community#faq) + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + engine.io-client@6.6.3: resolution: {integrity: sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==} @@ -2460,6 +2600,10 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} @@ -2644,6 +2788,9 @@ packages: resolution: {integrity: sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==} engines: {node: '>= 18'} + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2679,6 +2826,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -2722,6 +2873,10 @@ packages: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + form-data@4.0.4: resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} engines: {node: '>= 6'} @@ -2730,6 +2885,10 @@ packages: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -2774,10 +2933,22 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + generator-function@2.0.1: resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} engines: {node: '>= 0.4'} + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + get-east-asian-width@1.4.0: resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} engines: {node: '>=18'} @@ -2816,6 +2987,10 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + hasBin: true + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported @@ -2832,6 +3007,18 @@ packages: resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} engines: {node: '>= 0.4'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} + + google-gax@5.0.5: + resolution: {integrity: sha512-VuC6nVnPVfo/M1WudLoS4Y3dTepVndZatUmeb0nUNmfzft6mKSy8ffDh4h5qxR7L9lslDxNpWPYsuPrFboOmTw==} + engines: {node: '>=18'} + + google-logging-utils@1.1.2: + resolution: {integrity: sha512-YsFPGVgDFf4IzSwbwIR0iaFJQFmR5Jp7V1WuYSjuRgAm9yWqsMhKE9YPlL+wvFLnc/wMiFV4SQUD9Y/JMpxIxQ==} + engines: {node: '>=14'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -2842,6 +3029,10 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} + has-bigints@1.1.0: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} @@ -2873,6 +3064,10 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + heap-js@2.7.1: + resolution: {integrity: sha512-EQfezRg0NCZGNlhlDR3Evrw1FVL2G3LhU7EgPoxufQKruNBSYA8MiRPHeWbU+36o+Fhel0wMwM+sLEiBAlNLJA==} + engines: {node: '>=10.0.0'} + hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} @@ -2880,6 +3075,18 @@ packages: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} + http-proxy-agent@5.0.0: + resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} + engines: {node: '>= 6'} + + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + human-signals@5.0.0: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} @@ -2997,6 +3204,10 @@ packages: resolution: {integrity: sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==} engines: {node: '>= 0.4'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-fullwidth-code-point@4.0.0: resolution: {integrity: sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==} engines: {node: '>=12'} @@ -3044,6 +3255,9 @@ packages: resolution: {integrity: sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==} engines: {node: '>= 0.4'} + is-stream-ended@0.1.4: + resolution: {integrity: sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==} + is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3085,6 +3299,9 @@ packages: resolution: {integrity: sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==} engines: {node: '>= 0.4'} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jiti@2.6.1: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true @@ -3101,6 +3318,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -3128,9 +3348,15 @@ packages: jwa@1.4.2: resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + jws@4.0.0: + resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + kafkajs@2.2.4: resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==} engines: {node: '>=14.0.0'} @@ -3288,6 +3514,9 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} @@ -3312,6 +3541,9 @@ packages: lodash.once@4.1.1: resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + lodash.snakecase@4.1.1: + resolution: {integrity: sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==} + lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} @@ -3319,6 +3551,9 @@ packages: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -3326,6 +3561,9 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + ltgt@2.2.1: resolution: {integrity: sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==} @@ -3691,6 +3929,11 @@ packages: resolution: {integrity: sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==} engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-domexception@2.0.2: resolution: {integrity: sha512-Qf9vHK9c5MGgUXj8SnucCIS4oEPuUstjRaMplLGeZpbWMfNV1rvEcXuwoXfN51dUfD1b4muPHPQtCx/5Dj/QAA==} engines: {node: '>=16'} @@ -3705,6 +3948,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build@4.1.1: resolution: {integrity: sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==} hasBin: true @@ -3734,6 +3981,10 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + object-hash@3.0.0: + resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} + engines: {node: '>= 6'} + object-inspect@1.13.4: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} @@ -3785,6 +4036,10 @@ packages: resolution: {integrity: sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==} engines: {node: '>= 0.4'} + p-defer@3.0.0: + resolution: {integrity: sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==} + engines: {node: '>=8'} + p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} @@ -3793,6 +4048,9 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -3824,6 +4082,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} @@ -3883,6 +4145,14 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + proto3-json-serializer@3.0.4: + resolution: {integrity: sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==} + engines: {node: '>=18'} + + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -4028,6 +4298,10 @@ packages: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} engines: {node: '>= 0.4'} + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + requires-port@1.0.0: resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} @@ -4051,6 +4325,10 @@ packages: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} + retry-request@8.0.2: + resolution: {integrity: sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==} + engines: {node: '>=18'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -4063,6 +4341,10 @@ packages: deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + router@2.2.0: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} @@ -4229,6 +4511,12 @@ packages: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} + stream-events@1.0.5: + resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + + stream-shift@1.0.3: + resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} + streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} @@ -4237,6 +4525,14 @@ packages: resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} engines: {node: '>=0.6.19'} + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + string-width@7.2.0: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} @@ -4267,6 +4563,10 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + strip-ansi@7.1.2: resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} engines: {node: '>=12'} @@ -4287,6 +4587,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + stubs@3.0.0: + resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} + styled-jsx@5.1.6: resolution: {integrity: sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==} engines: {node: '>= 12.0.0'} @@ -4333,6 +4636,10 @@ packages: resolution: {integrity: sha512-nlGpxf+hv0v7GkWBK2V9spgactGOp0qvfWRxUMjqHyzrt3SgwE48DIv/FhqPHJYLHpgW1opq3nERbz5Anq7n1g==} engines: {node: '>=18'} + teeny-request@10.1.0: + resolution: {integrity: sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==} + engines: {node: '>=18'} + tinyglobby@0.2.15: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} @@ -4519,6 +4826,10 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -4558,6 +4869,14 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + wrap-ansi@9.0.2: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} @@ -4637,6 +4956,10 @@ packages: peerDependencies: yjs: ^13.5.6 + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yallist@5.0.0: resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} engines: {node: '>=18'} @@ -4650,6 +4973,14 @@ packages: engines: {node: '>= 14.6'} hasBin: true + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + yjs@13.6.27: resolution: {integrity: sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -5000,6 +5331,48 @@ snapshots: '@floating-ui/utils@0.2.10': {} + '@google-cloud/paginator@6.0.0': + dependencies: + extend: 3.0.2 + + '@google-cloud/precise-date@5.0.0': {} + + '@google-cloud/projectify@5.0.0': {} + + '@google-cloud/promisify@5.0.0': {} + + '@google-cloud/pubsub@5.2.0': + dependencies: + '@google-cloud/paginator': 6.0.0 + '@google-cloud/precise-date': 5.0.0 + '@google-cloud/projectify': 5.0.0 + '@google-cloud/promisify': 5.0.0 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.34.0 + arrify: 2.0.1 + extend: 3.0.2 + google-auth-library: 10.5.0 + google-gax: 5.0.5 + heap-js: 2.7.1 + is-stream-ended: 0.1.4 + lodash.snakecase: 4.1.1 + p-defer: 3.0.0 + transitivePeerDependencies: + - supports-color + + '@grpc/grpc-js@1.14.1': + dependencies: + '@grpc/proto-loader': 0.8.0 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.8.0': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.5.4 + yargs: 17.7.2 + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.7': @@ -5102,6 +5475,15 @@ snapshots: '@img/sharp-win32-x64@0.34.4': optional: true + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.2 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.2 @@ -5130,6 +5512,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@js-sdsl/ordered-map@4.4.2': {} + '@monaco-editor/loader@1.6.1': dependencies: state-local: 1.0.7 @@ -5287,10 +5671,47 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api@1.9.0': {} + + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.28.0 + + '@opentelemetry/semantic-conventions@1.28.0': {} + + '@opentelemetry/semantic-conventions@1.34.0': {} + + '@pkgjs/parseargs@0.11.0': + optional: true + '@pkgr/core@0.2.9': {} '@popperjs/core@2.11.8': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@radix-ui/number@1.1.1': {} '@radix-ui/primitive@1.1.3': {} @@ -5821,6 +6242,8 @@ snapshots: '@tanstack/table-core@8.21.3': {} + '@tootallnate/once@2.0.0': {} + '@tsconfig/node10@1.0.11': {} '@tsconfig/node12@1.0.11': {} @@ -6148,6 +6571,14 @@ snapshots: acorn@8.15.0: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + + agent-base@7.1.4: {} + agora-access-token@2.0.4: dependencies: crc-32: 1.2.0 @@ -6173,6 +6604,8 @@ snapshots: dependencies: environment: 1.1.0 + ansi-regex@5.0.1: {} + ansi-regex@6.2.2: {} ansi-styles@4.3.0: @@ -6267,6 +6700,8 @@ snapshots: get-intrinsic: 1.3.0 is-array-buffer: 3.0.5 + arrify@2.0.1: {} + ast-types-flow@0.0.8: {} async-function@1.0.0: {} @@ -6300,8 +6735,7 @@ snapshots: balanced-match@1.0.2: {} - base64-js@1.5.1: - optional: true + base64-js@1.5.1: {} base64id@2.0.0: {} @@ -6310,6 +6744,8 @@ snapshots: node-addon-api: 8.5.0 node-gyp-build: 4.8.4 + bignumber.js@9.3.1: {} + binary-extensions@2.3.0: {} body-parser@1.20.0: @@ -6437,6 +6873,12 @@ snapshots: client-only@0.0.1: {} + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + cloudinary@2.8.0: dependencies: lodash: 4.17.21 @@ -6540,6 +6982,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@4.0.1: {} + data-view-buffer@1.0.2: dependencies: call-bound: 1.0.4 @@ -6637,10 +7081,19 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + duplexify@4.1.3: + dependencies: + end-of-stream: 1.4.5 + inherits: 2.0.4 + readable-stream: 3.6.2 + stream-shift: 1.0.3 + dynamic-dedupe@0.3.0: dependencies: xtend: 4.0.2 + eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 @@ -6661,6 +7114,8 @@ snapshots: emoji-regex@10.6.0: {} + emoji-regex@8.0.0: {} + emoji-regex@9.2.2: {} encodeurl@1.0.2: {} @@ -6675,6 +7130,10 @@ snapshots: level-errors: 2.0.1 optional: true + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + engine.io-client@6.6.3: dependencies: '@socket.io/component-emitter': 3.1.2 @@ -6822,6 +7281,8 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + escalade@3.2.0: {} + escape-html@1.0.3: {} escape-string-regexp@4.0.0: {} @@ -7134,6 +7595,8 @@ snapshots: transitivePeerDependencies: - supports-color + extend@3.0.2: {} + fast-deep-equal@3.1.3: {} fast-diff@1.3.0: {} @@ -7170,6 +7633,11 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 @@ -7221,6 +7689,11 @@ snapshots: dependencies: is-callable: 1.2.7 + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + form-data@4.0.4: dependencies: asynckit: 0.4.0 @@ -7231,6 +7704,10 @@ snapshots: format@0.2.2: {} + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + forwarded@0.2.0: {} framer-motion@12.23.24(@emotion/is-prop-valid@1.4.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0): @@ -7265,8 +7742,27 @@ snapshots: functions-have-names@1.2.3: {} + gaxios@7.1.3: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.1.3 + google-logging-utils: 1.1.2 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + generator-function@2.0.1: {} + get-caller-file@2.0.5: {} + get-east-asian-width@1.4.0: {} get-intrinsic@1.3.0: @@ -7311,6 +7807,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.4.5: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -7329,12 +7834,49 @@ snapshots: define-properties: 1.2.1 gopd: 1.2.0 + google-auth-library@10.5.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.3 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.2 + gtoken: 8.0.0 + jws: 4.0.0 + transitivePeerDependencies: + - supports-color + + google-gax@5.0.5: + dependencies: + '@grpc/grpc-js': 1.14.1 + '@grpc/proto-loader': 0.8.0 + duplexify: 4.1.3 + google-auth-library: 10.5.0 + google-logging-utils: 1.1.2 + node-fetch: 3.3.2 + object-hash: 3.0.0 + proto3-json-serializer: 3.0.4 + protobufjs: 7.5.4 + retry-request: 8.0.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.2: {} + gopd@1.2.0: {} graceful-fs@4.2.11: {} graphemer@1.4.0: {} + gtoken@8.0.0: + dependencies: + gaxios: 7.1.3 + jws: 4.0.0 + transitivePeerDependencies: + - supports-color + has-bigints@1.1.0: {} has-flag@3.0.0: {} @@ -7359,6 +7901,8 @@ snapshots: dependencies: function-bind: 1.1.2 + heap-js@2.7.1: {} + hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1 @@ -7371,6 +7915,28 @@ snapshots: statuses: 2.0.1 toidentifier: 1.0.1 + http-proxy-agent@5.0.0: + dependencies: + '@tootallnate/once': 2.0.0 + agent-base: 6.0.2 + debug: 4.4.3(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + human-signals@5.0.0: {} husky@9.1.7: {} @@ -7482,6 +8048,8 @@ snapshots: dependencies: call-bound: 1.0.4 + is-fullwidth-code-point@3.0.0: {} + is-fullwidth-code-point@4.0.0: {} is-fullwidth-code-point@5.1.0: @@ -7526,6 +8094,8 @@ snapshots: dependencies: call-bound: 1.0.4 + is-stream-ended@0.1.4: {} + is-stream@3.0.0: {} is-string@1.1.1: @@ -7569,6 +8139,12 @@ snapshots: has-symbols: 1.1.0 set-function-name: 2.0.2 + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jiti@2.6.1: {} js-tokens@4.0.0: {} @@ -7579,6 +8155,10 @@ snapshots: jsesc@3.1.0: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + json-buffer@3.0.1: {} json-parse-even-better-errors@2.3.1: {} @@ -7617,11 +8197,22 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + jws@3.2.2: dependencies: jwa: 1.4.2 safe-buffer: 5.2.1 + jws@4.0.0: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + kafkajs@2.2.4: {} kareem@2.6.3: {} @@ -7784,6 +8375,8 @@ snapshots: dependencies: p-locate: 5.0.0 + lodash.camelcase@4.3.0: {} + lodash.includes@4.3.0: {} lodash.isboolean@3.0.3: {} @@ -7800,6 +8393,8 @@ snapshots: lodash.once@4.1.1: {} + lodash.snakecase@4.1.1: {} + lodash@4.17.21: {} log-update@6.1.0: @@ -7810,12 +8405,16 @@ snapshots: strip-ansi: 7.1.2 wrap-ansi: 9.0.2 + long@5.3.2: {} + longest-streak@3.1.0: {} loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 + lru-cache@10.4.3: {} + ltgt@2.2.1: optional: true @@ -8294,7 +8893,7 @@ snapshots: negotiator@1.0.0: {} - next@15.5.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0): + next@15.5.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0): dependencies: '@next/env': 15.5.4 '@swc/helpers': 0.5.15 @@ -8312,6 +8911,7 @@ snapshots: '@next/swc-linux-x64-musl': 15.5.4 '@next/swc-win32-arm64-msvc': 15.5.4 '@next/swc-win32-x64-msvc': 15.5.4 + '@opentelemetry/api': 1.9.0 sharp: 0.34.4 transitivePeerDependencies: - '@babel/core' @@ -8319,12 +8919,20 @@ snapshots: node-addon-api@8.5.0: {} + node-domexception@1.0.0: {} + node-domexception@2.0.2: {} node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-gyp-build@4.1.1: optional: true @@ -8353,6 +8961,8 @@ snapshots: object-assign@4.1.1: {} + object-hash@3.0.0: {} + object-inspect@1.13.4: {} object-keys@1.1.1: {} @@ -8424,6 +9034,8 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 + p-defer@3.0.0: {} + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 @@ -8432,6 +9044,8 @@ snapshots: dependencies: p-limit: 3.1.0 + package-json-from-dist@1.0.1: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -8455,6 +9069,11 @@ snapshots: path-parse@1.0.7: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + path-to-regexp@0.1.7: {} path-to-regexp@8.3.0: {} @@ -8499,6 +9118,25 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 + proto3-json-serializer@3.0.4: + dependencies: + protobufjs: 7.5.4 + + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 20.19.21 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -8647,6 +9285,8 @@ snapshots: gopd: 1.2.0 set-function-name: 2.0.2 + require-directory@2.1.1: {} + requires-port@1.0.0: {} resolve-from@4.0.0: {} @@ -8670,6 +9310,13 @@ snapshots: onetime: 7.0.0 signal-exit: 4.1.0 + retry-request@8.0.2: + dependencies: + extend: 3.0.2 + teeny-request: 10.1.0 + transitivePeerDependencies: + - supports-color + reusify@1.1.0: {} rfdc@1.4.1: {} @@ -8678,6 +9325,10 @@ snapshots: dependencies: glob: 7.2.3 + rimraf@5.0.10: + dependencies: + glob: 10.4.5 + router@2.2.0: dependencies: debug: 4.4.3(supports-color@5.5.0) @@ -8948,10 +9599,28 @@ snapshots: es-errors: 1.3.0 internal-slot: 1.1.0 + stream-events@1.0.5: + dependencies: + stubs: 3.0.0 + + stream-shift@1.0.3: {} + streamsearch@1.1.0: {} string-argv@0.3.2: {} + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.2 + string-width@7.2.0: dependencies: emoji-regex: 10.6.0 @@ -9012,6 +9681,10 @@ snapshots: dependencies: safe-buffer: 5.2.1 + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + strip-ansi@7.1.2: dependencies: ansi-regex: 6.2.2 @@ -9024,6 +9697,8 @@ snapshots: strip-json-comments@3.1.1: {} + stubs@3.0.0: {} + styled-jsx@5.1.6(react@19.1.0): dependencies: client-only: 0.0.1 @@ -9059,6 +9734,15 @@ snapshots: minizlib: 3.1.0 yallist: 5.0.0 + teeny-request@10.1.0: + dependencies: + http-proxy-agent: 5.0.0 + https-proxy-agent: 5.0.1 + node-fetch: 3.3.2 + stream-events: 1.0.5 + transitivePeerDependencies: + - supports-color + tinyglobby@0.2.15: dependencies: fdir: 6.5.0(picomatch@4.0.3) @@ -9291,6 +9975,8 @@ snapshots: vary@1.1.2: {} + web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@7.0.0: {} @@ -9352,6 +10038,18 @@ snapshots: word-wrap@1.2.5: {} + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.1.2 + wrap-ansi@9.0.2: dependencies: ansi-styles: 6.2.3 @@ -9411,12 +10109,26 @@ snapshots: y-protocols: 1.0.6(yjs@13.6.27) yjs: 13.6.27 + y18n@5.0.8: {} + yallist@5.0.0: {} yaml@1.10.2: {} yaml@2.8.1: {} + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + yjs@13.6.27: dependencies: lib0: 0.2.114 From 2ec521269fd200ef4d005a7c7391d0bc3c84a019 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Sat, 8 Nov 2025 16:31:58 +0800 Subject: [PATCH 08/12] feat(deployment): add terraform build instructions for gcp --- .gitignore | 46 +++ scripts/build-and-push.sh | 81 +++++ terraform/.terraform.lock.hcl | 22 ++ terraform/loadbalancer.tf | 197 +++++++++++ terraform/main.tf | 540 +++++++++++++++++++++++++++++ terraform/outputs.tf | 34 ++ terraform/provider.tf | 20 ++ terraform/terraform.tfvars.example | 4 + terraform/variables.tf | 22 ++ 9 files changed, 966 insertions(+) create mode 100755 scripts/build-and-push.sh create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/loadbalancer.tf create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/provider.tf create mode 100644 terraform/terraform.tfvars.example create mode 100644 terraform/variables.tf diff --git a/.gitignore b/.gitignore index 11f274e5aa..d5cb76e07d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,49 @@ build coverage .next .env +.DS_Store + +# Local .terraform directories +.terraform/ + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Optional: ignore graph output files generated by `terraform graph` +# *.dot + +# Optional: ignore plan files saved before destroying Terraform configuration +# Uncomment the line below if you want to ignore planout files. +# planout \ No newline at end of file diff --git a/scripts/build-and-push.sh b/scripts/build-and-push.sh new file mode 100755 index 0000000000..0df2e0faf6 --- /dev/null +++ b/scripts/build-and-push.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Build and push Docker images to Google Artifact Registry +# Usage: ./scripts/build-and-push.sh + +set -e + +PROJECT_ID=${1:-"gcp-project-id"} +REGION=${2:-"gcp-region"} +REGISTRY="${REGION}-docker.pkg.dev/${PROJECT_ID}/docker-repo" + +# Static service URLs for frontend build args +DOMAIN="cs3219-ay2526s1-g13.com" +USER_SERVICE_URL="https://${DOMAIN}/api/user" +MATCHING_SERVICE_URL="https://${DOMAIN}/api/matching" +QUESTION_SERVICE_URL="https://${DOMAIN}/api/question" +COLLABORATION_SERVICE_URL="https://${DOMAIN}/api/collaboration" + + +echo "🔨 Building and pushing Docker images to ${REGISTRY}" + +# Authenticate with Google Cloud +echo "🔐 Authenticating with Google Cloud..." +gcloud auth configure-docker ${REGION}-docker.pkg.dev + +# Services to build (service-name:context-path:dockerfile-name) +SERVICES=( + "user-service:backend/user-service:Dockerfile.user" + "matching-service:backend/matching-service:Dockerfile.matching" + "question-service:backend/question-service:Dockerfile.question" + "collaboration-service:backend/collaboration-service:Dockerfile.collaboration" + "execution-service:backend/execution-service:Dockerfile.execution" + "frontend:frontend:Dockerfile.frontend" +) + +for SERVICE_CONFIG in "${SERVICES[@]}"; do + IFS=':' read -r SERVICE_NAME SERVICE_PATH DOCKERFILE <<< "$SERVICE_CONFIG" + + echo "" + echo "📦 Building ${SERVICE_NAME}..." + + # Check if Dockerfile exists + if [ ! -f "${SERVICE_PATH}/${DOCKERFILE}" ]; then + echo "❌ Error: ${DOCKERFILE} not found at ${SERVICE_PATH}/${DOCKERFILE}" + echo " Please ensure the Dockerfile exists before building." + exit 1 + fi + + # Need to build for linux/amd64 for GCP compatibility + echo " Using ${DOCKERFILE}" + + # Use an if-statement to provide build ARGs ONLY for the frontend + if [ "$SERVICE_NAME" == "frontend" ]; then + echo " Injecting build arguments for frontend..." + docker buildx build --platform linux/amd64 \ + --build-arg USER_SERVICE_URL="$USER_SERVICE_URL" \ + --build-arg MATCHING_SERVICE_URL="$MATCHING_SERVICE_URL" \ + --build-arg QUESTION_SERVICE_URL="$QUESTION_SERVICE_URL" \ + --build-arg COLLABORATION_SERVICE_URL="$COLLABORATION_SERVICE_URL" \ + -f "${SERVICE_PATH}/${DOCKERFILE}" -t "${REGISTRY}/${SERVICE_NAME}:latest" --load "${SERVICE_PATH}" + else + docker buildx build --platform linux/amd64 \ + -f "${SERVICE_PATH}/${DOCKERFILE}" -t "${REGISTRY}/${SERVICE_NAME}:latest" --load "${SERVICE_PATH}" + fi + + # Push image + echo "⬆️ Pushing ${SERVICE_NAME}..." + docker push "${REGISTRY}/${SERVICE_NAME}:latest" + + echo "✅ ${SERVICE_NAME} pushed successfully" +done + +echo "" +echo "🎉 All images built and pushed successfully!" +echo " Registry: ${REGISTRY}" +echo "" +echo "Images pushed:" +for SERVICE_CONFIG in "${SERVICES[@]}"; do + IFS=':' read -r SERVICE_NAME SERVICE_PATH DOCKERFILE <<< "$SERVICE_CONFIG" + echo " - ${REGISTRY}/${SERVICE_NAME}:latest" +done diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000000..a3fc489d0c --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.10.0" + constraints = "~> 7.10" + hashes = [ + "h1:itPAvjmGmV6R8KpBsDGnONKfi9AXPt0LCFThnDF/3+g=", + "zh:40b90c1e6cc95cccfc935d4af4478590410bf2ce24d2cd7f6b583e7791b9b5b3", + "zh:56f25840e253ea527ab196ffb5cc3f896ef129c6d0dbe795eb9dd305d84354cd", + "zh:5bb291ae271b3363052ea4b01073c91ce9a5fcf58bf8c1d01a919099e1b4e946", + "zh:671daceb89669b51586a1a32861ea77d30233d6c2adb1ec2d3ada48d3f485634", + "zh:965834aeda62b59b8140b8db9d378658d5e3bb56a828a5158d2625237ae925b5", + "zh:9a83c890f65bcd3777eb42a9a7de622c6f9ceb3fe3ceedfe532c1cb4de24ddb2", + "zh:a2a995d03f8a669a753e99e12e6826c0b7081ac573a66c9c90db2d70f00b8ed4", + "zh:a4473ea1f59b6a5837f5e27ac4383a566dc8bdf60e0e63b5fddc5a2523fdaeb5", + "zh:a568e0b3c475629de3d00f60693270d7ae2d8cda7a9b8b28613baec9fb09b37c", + "zh:c53002313ea07dba92503eec6c0876b13b5217a703313c3ac74b31fefa52afbb", + "zh:dbd8c9e2099ca321ebab124b9c9a3421469714ac2d329d0047d0981c8c64ab21", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/loadbalancer.tf b/terraform/loadbalancer.tf new file mode 100644 index 0000000000..b1897cd33f --- /dev/null +++ b/terraform/loadbalancer.tf @@ -0,0 +1,197 @@ +# Reserve a global static IP address for the load balancer +resource "google_compute_global_address" "lb_ip" { + name = "peerprep-lb-ip" +} + +# Create Serverless Network Endpoint Groups (NEGs) for each Cloud Run service +resource "google_compute_region_network_endpoint_group" "user_service_neg" { + name = "user-service-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.user_service.name + } +} + +resource "google_compute_region_network_endpoint_group" "question_service_neg" { + name = "question-service-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.question_service.name + } +} + +resource "google_compute_region_network_endpoint_group" "matching_service_neg" { + name = "matching-service-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.matching_service.name + } +} + +resource "google_compute_region_network_endpoint_group" "collab_service_neg" { + name = "collab-service-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.collaboration_service.name + } +} + +resource "google_compute_region_network_endpoint_group" "frontend_neg" { + name = "frontend-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.frontend.name + } +} + +# Create Backend Services for each NEG +resource "google_compute_backend_service" "user_service_backend" { + name = "user-service-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.user_service_neg.id + } +} + +resource "google_compute_backend_service" "question_service_backend" { + name = "question-service-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.question_service_neg.id + } +} + +resource "google_compute_backend_service" "matching_service_backend" { + name = "matching-service-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.matching_service_neg.id + } +} + +resource "google_compute_backend_service" "collab_service_backend" { + name = "collab-service-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.collab_service_neg.id + } +} + +resource "google_compute_backend_service" "frontend_backend" { + name = "frontend-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.frontend_neg.id + } +} + + +# Create URL Map to route traffic based on paths +resource "google_compute_url_map" "api_url_map" { + name = "peerprep-api-url-map" + # Route all root traffic to the frontend + default_service = google_compute_backend_service.frontend_backend.id + + host_rule { + hosts = ["cs3219-ay2526s1-g13.com"] + path_matcher = "api-paths" + } + + path_matcher { + name = "api-paths" + # Route all root traffic to the frontend + default_service = google_compute_backend_service.frontend_backend.id + + # Custom route rules for each API service + route_rules { + priority = 1 + match_rules { + prefix_match = "/api/user/" + } + service = google_compute_backend_service.user_service_backend.id + route_action { + url_rewrite { + path_prefix_rewrite = "/" + } + } + } + + route_rules { + priority = 2 + match_rules { + prefix_match = "/api/question/" + } + service = google_compute_backend_service.question_service_backend.id + route_action { + url_rewrite { + path_prefix_rewrite = "/" + } + } + } + + route_rules { + priority = 3 + match_rules { + prefix_match = "/api/matching/" + } + service = google_compute_backend_service.matching_service_backend.id + route_action { + url_rewrite { + path_prefix_rewrite = "/" + } + } + } + + route_rules { + priority = 4 + match_rules { + prefix_match = "/api/collaboration/" + } + service = google_compute_backend_service.collab_service_backend.id + route_action { + url_rewrite { + path_prefix_rewrite = "/" + } + } + } + } +} + +# Create a Google-managed SSL certificate +resource "google_compute_managed_ssl_certificate" "api_ssl" { + name = "peerprep-ssl-cert" + managed { + domains = ["cs3219-ay2526s1-g13.com"] + } +} + +# Create the HTTPS proxy +resource "google_compute_target_https_proxy" "api_proxy" { + name = "peerprep-https-proxy" + url_map = google_compute_url_map.api_url_map.id + ssl_certificates = [google_compute_managed_ssl_certificate.api_ssl.id] +} + +# Create the Global Forwarding Rule (Connects the IP to the proxy) +resource "google_compute_global_forwarding_rule" "api_forwarding_rule" { + name = "peerprep-forwarding-rule" + ip_protocol = "TCP" + port_range = "443" + ip_address = google_compute_global_address.lb_ip.address + target = google_compute_target_https_proxy.api_proxy.id +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000..d88f3914e8 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,540 @@ +# Enable required APIs +resource "google_project_service" "apis" { + for_each = toset([ + "run.googleapis.com", + "redis.googleapis.com", + "pubsub.googleapis.com", + "artifactregistry.googleapis.com", + "vpcaccess.googleapis.com", + "compute.googleapis.com", + ]) + + service = each.key + disable_on_destroy = false +} + +# Service Account for application +resource "google_service_account" "service_account" { + account_id = "peerprep-service-account" + display_name = "PeerPrep Service Account" +} + +# IAM Binding for Service Account to access Secret Manager +resource "google_project_iam_member" "secret_accessor" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +# IAM Binding for Service Account to access Pub/Sub +resource "google_project_iam_member" "pubsub_editor" { + project = var.project_id + role = "roles/pubsub.editor" + member = "serviceAccount:${google_service_account.service_account.email}" +} + +# VPC for private services +resource "google_compute_network" "vpc" { + name = "peerprep-vpc" + auto_create_subnetworks = false + depends_on = [google_project_service.apis] +} + +resource "google_compute_subnetwork" "subnet" { + name = "peerprep-subnet" + ip_cidr_range = "10.0.0.0/24" + region = var.region + network = google_compute_network.vpc.id +} + +# Serverless VPC Access Connector (for Cloud Run to access Redis) +resource "google_vpc_access_connector" "connector" { + name = "peerprep-connector" + region = var.region + network = google_compute_network.vpc.id + ip_cidr_range = "10.8.0.0/28" + machine_type = "e2-micro" + min_instances = 2 + max_instances = 5 + + depends_on = [google_project_service.apis] +} + +# Redis (Memorystore) +resource "google_redis_instance" "redis" { + name = "peerprep-redis" + tier = "BASIC" + memory_size_gb = 1 + region = var.region + + authorized_network = google_compute_network.vpc.id + + redis_version = "REDIS_7_0" + display_name = "PeerPrep Redis" + + depends_on = [google_project_service.apis] +} + +# Pub/Sub Topics (Kafka replacement) +resource "google_pubsub_topic" "match_topic" { + name = "match_topic" + + depends_on = [google_project_service.apis] +} + +resource "google_pubsub_topic" "question_topic" { + name = "question_topic" + + depends_on = [google_project_service.apis] +} + +resource "google_pubsub_topic" "room_creation_topic" { + name = "room_creation_topic" + + depends_on = [google_project_service.apis] +} + +resource "google_pubsub_topic" "room_created_topic" { + name = "room_created_topic" + + depends_on = [google_project_service.apis] +} + +# Pub/Sub Subscriptions (Kafka replacement) +resource "google_pubsub_subscription" "match_sub" { + name = "match_sub" + topic = google_pubsub_topic.match_topic.name + + ack_deadline_seconds = 20 + + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } +} + +resource "google_pubsub_subscription" "question_sub" { + name = "question_sub" + topic = google_pubsub_topic.question_topic.name + + ack_deadline_seconds = 20 + + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } +} + +resource "google_pubsub_subscription" "room_creation_sub" { + name = "room_creation_sub" + topic = google_pubsub_topic.room_creation_topic.name + + ack_deadline_seconds = 20 + + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } +} + +resource "google_pubsub_subscription" "room_created_sub" { + name = "room_created_sub" + topic = google_pubsub_topic.room_created_topic.name + + ack_deadline_seconds = 20 + + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } +} + +# Artifact Registry for Docker images +resource "google_artifact_registry_repository" "docker_repo" { + location = var.region + repository_id = "docker-repo" + description = "Docker Repository" + format = "DOCKER" + + depends_on = [google_project_service.apis] +} + +# Cloud Run Service - User Service +resource "google_cloud_run_v2_service" "user_service" { + name = "user-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/user-service:latest" + + ports { + container_port = 8001 + } + + env { + name = "MONGO_URI" + value_source { + secret_key_ref { + secret = "mongodb_uri_user" + version = "latest" + } + } + } + + env { + name = "WEB_BASE_URL" + value = "https://cs3219-ay2526s1-g13.com" + } + + env { + name = "EMAIL_USER" + value_source { + secret_key_ref { + secret = "email_user" + version = "latest" + } + } + } + + env { + name = "EMAIL_PASSWORD" + value_source { + secret_key_ref { + secret = "email_password" + version = "latest" + } + } + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = "jwt_secret" + version = "latest" + } + } + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + +# Cloud Run Service - Question Service +resource "google_cloud_run_v2_service" "question_service" { + name = "question-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/question-service:latest" + + ports { + container_port = 8003 + } + + env { + name = "MONGO_URI" + value_source { + secret_key_ref { + secret = "mongodb_uri_question" + version = "latest" + } + } + } + + env { + name = "WEB_BASE_URL" + value = "https://cs3219-ay2526s1-g13.com" + } + + env { + name = "PUBSUB_PROJECT_ID" + value = var.project_id + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + +# Cloud Run Service - Matching Service +resource "google_cloud_run_v2_service" "matching_service" { + name = "matching-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/matching-service:latest" + + ports { + container_port = 8002 + } + + env { + name = "NODE_ENV" + value = var.environment + } + + env { + name = "REDIS_URL" + value = "redis://${google_redis_instance.redis.host}:${google_redis_instance.redis.port}" + } + + env { + name = "QUESTION_SERVICE_URL" + value = google_cloud_run_v2_service.question_service.uri + } + + env { + name = "COLLABORATION_SERVICE_URL" + value = google_cloud_run_v2_service.collaboration_service.uri + } + + env { + name = "DEBUG_MODE" + value = "false" + } + env { + name = "DEBUG_MATCHING" + value = "false" + } + + env { + name = "PUBSUB_PROJECT_ID" + value = var.project_id + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = "jwt_secret" + version = "latest" + } + } + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + + vpc_access { + connector = google_vpc_access_connector.connector.id + egress = "PRIVATE_RANGES_ONLY" + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + +# Cloud Run Service - Collaboration Service +resource "google_cloud_run_v2_service" "collaboration_service" { + name = "collaboration-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/collaboration-service:latest" + + ports { + container_port = 8004 + } + + env { + name = "MONGODB_URI" + value_source { + secret_key_ref { + secret = "mongodb_uri_collaboration" + version = "latest" + } + } + } + + env { + name = "WEB_BASE_URL" + value = "https://cs3219-ay2526s1-g13.com" + } + + env { + name = "ROOM_TIMEOUT_MINUTES" + value = var.room_timeout_minutes + } + + env { + name = "QUESTION_SERVICE_URL" + value = google_cloud_run_v2_service.question_service.uri + } + + env { + name = "PUBSUB_PROJECT_ID" + value = var.project_id + } + + env { + name = "JWT_SECRET" + value_source { + secret_key_ref { + secret = "jwt_secret" + version = "latest" + } + } + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + +# Cloud Run Service - Frontend +resource "google_cloud_run_v2_service" "frontend" { + name = "frontend" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/frontend:latest" + + ports { + container_port = 3000 + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + +# IAM - Allow public access to services +resource "google_cloud_run_service_iam_member" "user_service_public" { + service = google_cloud_run_v2_service.user_service.name + location = google_cloud_run_v2_service.user_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service_iam_member" "question_service_public" { + service = google_cloud_run_v2_service.question_service.name + location = google_cloud_run_v2_service.question_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service_iam_member" "matching_service_public" { + service = google_cloud_run_v2_service.matching_service.name + location = google_cloud_run_v2_service.matching_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service_iam_member" "collaboration_service_public" { + service = google_cloud_run_v2_service.collaboration_service.name + location = google_cloud_run_v2_service.collaboration_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service_iam_member" "frontend_public" { + service = google_cloud_run_v2_service.frontend.name + location = google_cloud_run_v2_service.frontend.location + role = "roles/run.invoker" + member = "allUsers" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000000..4807d7acd7 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,34 @@ +output "static_ip" { + description = "Static IP address for the load balancer" + value = google_compute_global_address.lb_ip.address +} + +output "frontend_url" { + description = "Frontend application URL" + value = google_cloud_run_v2_service.frontend.uri +} + +output "user_service_url" { + description = "User service URL" + value = google_cloud_run_v2_service.user_service.uri +} + +output "matching_service_url" { + description = "Matching service URL" + value = google_cloud_run_v2_service.matching_service.uri +} + +output "question_service_url" { + description = "Question service URL" + value = google_cloud_run_v2_service.question_service.uri +} + +output "collaboration_service_url" { + description = "Collaboration service URL" + value = google_cloud_run_v2_service.collaboration_service.uri +} + +output "redis_host" { + description = "Redis host address" + value = google_redis_instance.redis.host +} diff --git a/terraform/provider.tf b/terraform/provider.tf new file mode 100644 index 0000000000..5b5a7cfe7b --- /dev/null +++ b/terraform/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.10" + } + } + + backend "gcs" { + bucket = "cs3219-peerprep-terraform-state" + prefix = "terraform/state" + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100644 index 0000000000..a36afbea7b --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,4 @@ +project_id = +region = +environment = "production" +room_timeout_minutes = 10 diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..991e071051 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,22 @@ +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "asia-southeast1" +} + +variable "environment" { + description = "Environment name (dev, staging, production)" + type = string + default = "production" +} + +variable "room_timeout_minutes" { + description = "Collaboration service room timeout in minutes" + type = number + default = 10 +} From c3dc8049a6b6d99044e37adce04bfdfeb4860d20 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Tue, 11 Nov 2025 23:38:11 +0800 Subject: [PATCH 09/12] chore(mongodb): update user-service db name to match convention --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 68a6abb3b1..1eafd7974f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,7 @@ services: env_file: - ./backend/user-service/.env environment: - MONGO_URI: mongodb://admin:password@mongodb:27017/peerprepUserDB?authSource=admin + MONGO_URI: mongodb://admin:password@mongodb:27017/peerprepUserServiceDB?authSource=admin WEB_BASE_URL: http://localhost:3000 PORT: 8001 networks: From 6149d4fac13f23565f3eb92a044a694f3bdf79a8 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Tue, 11 Nov 2025 23:40:36 +0800 Subject: [PATCH 10/12] feat(rabbitmq): add gcp pubsub as alternative --- .../src/config/pubsub.js | 33 +- .../src/config/rabbitmq.js | 80 ++++ .../src/http/codeExecutionRoutes.js | 235 ++++++------ backend/collaboration-service/src/server.js | 6 +- backend/execution-service/.env.example | 2 +- backend/execution-service/package.json | 7 +- backend/execution-service/src/index.js | 33 ++ backend/execution-service/src/jobProcessor.js | 147 ++++++++ backend/execution-service/src/pubsubWorker.js | 78 ++++ .../execution-service/src/rabbitmqWorker.js | 61 +++ backend/execution-service/src/worker.js | 349 +++++++++--------- docker-compose.yml | 2 +- pnpm-lock.yaml | 3 + 13 files changed, 728 insertions(+), 308 deletions(-) create mode 100644 backend/collaboration-service/src/config/rabbitmq.js create mode 100644 backend/execution-service/src/index.js create mode 100644 backend/execution-service/src/jobProcessor.js create mode 100644 backend/execution-service/src/pubsubWorker.js create mode 100644 backend/execution-service/src/rabbitmqWorker.js diff --git a/backend/collaboration-service/src/config/pubsub.js b/backend/collaboration-service/src/config/pubsub.js index 0427c4b891..762d5d5bc4 100644 --- a/backend/collaboration-service/src/config/pubsub.js +++ b/backend/collaboration-service/src/config/pubsub.js @@ -1,10 +1,13 @@ import { PubSub } from "@google-cloud/pubsub"; +// Room creation topics export const ROOM_CREATION_TOPIC = "room_creation_topic"; export const ROOM_CREATED_TOPIC = "room_created_topic"; - export const ROOM_CREATION_SUBSCRIPTION = "room_creation_sub"; +// Code execution topics +export const JOB_EXECUTION_TOPIC = "job_execution_topic"; + export class PubSubManager { constructor() { console.log("Collaboration Service PubSubManager constructor"); @@ -46,7 +49,11 @@ export class PubSubManager { console.log(`Connected to Pub/Sub. Found ${topics.length} topics`); // Ensure topics exist - await this.ensureTopicsExist([ROOM_CREATION_TOPIC, ROOM_CREATED_TOPIC]); + await this.ensureTopicsExist([ + ROOM_CREATION_TOPIC, + ROOM_CREATED_TOPIC, + JOB_EXECUTION_TOPIC, + ]); this.isConnected = true; console.log("Connected to Pub/Sub"); @@ -154,6 +161,28 @@ export class PubSubManager { } } + async publishJob(job) { + if (!this.useGcp) { + console.log(`[Fallback] Would publish job: ${job.room_id}`); + return; + } + + try { + const topic = this.pubsub.topic(JOB_EXECUTION_TOPIC); + const dataBuffer = Buffer.from(JSON.stringify(job)); + + await topic.publishMessage({ + data: dataBuffer, + attributes: { room_id: job.room_id }, + }); + + console.log(">>> Sent job: ", job.room_id); + } catch (error) { + console.error("Error publishing job:", error); + throw error; + } + } + async disconnect() { if (!this.useGcp) { return; diff --git a/backend/collaboration-service/src/config/rabbitmq.js b/backend/collaboration-service/src/config/rabbitmq.js new file mode 100644 index 0000000000..bb631e44ff --- /dev/null +++ b/backend/collaboration-service/src/config/rabbitmq.js @@ -0,0 +1,80 @@ +import amqp from "amqplib"; + +export const JOB_EXECUTION_QUEUE = "job_execution_queue"; + +export class RabbitMQManager { + constructor() { + console.log("Collaboration Service RabbitMQManager constructor"); + const isGCP = !!process.env.PUBSUB_PROJECT_ID; + if (isGCP) { + console.log("GCP environment detected, RabbitMQ not configured for GCP"); + this.connection = null; + this.channel = null; + return; + } + this.rabbitMQUrl = process.env.RABBITMQ_URL || "amqp://user:password@rabbitmq"; + this.connection = null; + this.channel = null; + this.isConnected = false; + } + + async initWithRetry(maxRetries = 5, retryDelayMs = 2000) { + if (this.isConnected) { + console.log("Already connected to RabbitMQ"); + return; + } + + let attempt = 0; + while (attempt < maxRetries) { + try { + this.connection = await amqp.connect(this.rabbitMQUrl); + this.channel = await this.connection.createChannel(); + + await this.channel.assertQueue(JOB_EXECUTION_QUEUE, { durable: true }); + + console.log("Connected to RabbitMQ"); + this.isConnected = true; + return; + } catch (err) { + attempt += 1; + console.error(`Failed to connect to RabbitMQ (attempt ${attempt} of ${maxRetries}):`, err); + if (attempt >= maxRetries) { + throw err; + } + await new Promise((resolve) => setTimeout(resolve, retryDelayMs * attempt)); + } + } + } + + async sendJob(job) { + if (!this.isConnected || !this.channel) { + throw new Error("RabbitMQ not connected"); + } + + try { + this.channel.sendToQueue(JOB_EXECUTION_QUEUE, Buffer.from(JSON.stringify(job)), { + persistent: true, + }); + console.log(">>> Sent job: ", job.room_id); + } catch (error) { + console.error("Error while sending job: ", error.message); + throw error; + } + } + + async disconnect() { + try { + if (this.channel) { + await this.channel.close(); + } + if (this.connection) { + await this.connection.close(); + } + console.log("Disconnected from RabbitMQ"); + } catch (error) { + console.error("Error disconnecting from RabbitMQ:", error); + } + } +} + +export const rabbitmqManager = new RabbitMQManager(); diff --git a/backend/collaboration-service/src/http/codeExecutionRoutes.js b/backend/collaboration-service/src/http/codeExecutionRoutes.js index f48d978810..7ee0f34d1e 100644 --- a/backend/collaboration-service/src/http/codeExecutionRoutes.js +++ b/backend/collaboration-service/src/http/codeExecutionRoutes.js @@ -1,46 +1,33 @@ -import express from 'express' -import amqp from 'amqplib' -import roomManager from '../websocket/roomManager.js' +import express from "express"; +import roomManager from "../websocket/roomManager.js"; +import { rabbitmqManager } from "../config/rabbitmq.js"; +import { pubsubManager } from "../config/pubsub.js"; const router = express.Router(); -const RABBITMQ_URL = 'amqp://user:password@rabbitmq'; -const QUEUE_NAME = 'execution_jobs'; +const EXECUTION_TIMEOUT_MS = 60000; // 60s +const activeTimers = new Map(); +const useGcp = !!process.env.PUBSUB_PROJECT_ID; -const EXECUTION_TIMEOUT_MS = 60000 // 60s -const activeTimers = new Map() - -let mqChannel = null - - -export async function initRabbitMQ() { - try { - const connection = await amqp.connect(RABBITMQ_URL) - mqChannel = await connection.createChannel() - - await mqChannel.assertQueue(QUEUE_NAME, {durable: true}) - - console.log("RabbitMQ Producer connected") - } catch (error) { - console.log("Producer can't connect to RabbitMQ") - console.log(error.message) - process.exit(1) - } +export async function initCodeExecution() { + if (useGcp) { + await pubsubManager.initWithRetry(); + } else { + await rabbitmqManager.initWithRetry(); + } } async function sendJob(job) { - let connection - try { - mqChannel.sendToQueue( - QUEUE_NAME, - Buffer.from(JSON.stringify(job)), - {persistent: true} - ) - - console.log('>>> Sent job: ', job.room_id) - } catch (error) { - console.log ("Error while sending job: ", error.message) + try { + if (useGcp) { + await pubsubManager.publishJob(job); + } else { + await rabbitmqManager.sendJob(job); } + } catch (error) { + console.log("Error while sending job: ", error.message); + throw error; + } } /** @@ -49,57 +36,56 @@ async function sendJob(job) { * get code from FE and push a job to MQ */ router.post("/submit-code", async (req, res) => { - try { - // export job - const {room_id, language, source_code} = req.body - const job = { - room_id: room_id, - language: language, - source_code: source_code - } - - console.log(">>> Got code ", job) - roomManager.broadcastToRoom(room_id, { - type: "code-execution-started" - }) - - // delete old timers - if (activeTimers.has(room_id)) { - clearTimeout(activeTimers.get(room_id)) - activeTimers.delete(room_id) - } - - const newTimer = setTimeout(() => { - console.log('>>> Job timeout set for ', room_id) - - activeTimers.delete(room_id) - - const timeoutMessage = { - type: "code-execution-result", - data: { - isError: true, - output: "Execution timed out. Please try again" - } - } - - roomManager.broadcastToRoom(room_id, timeoutMessage) - }, EXECUTION_TIMEOUT_MS) - - activeTimers.set(room_id, newTimer) - - // send job to MQ - await sendJob(job) - res.status(202).json({ - message: "Job accepted" - }) - - } catch (error) { - console.error('Error during submission process:', error.message) - res.status(500).json({ - error: 'Internal Server Error' - }) + try { + // export job + const { room_id, language, source_code } = req.body; + const job = { + room_id: room_id, + language: language, + source_code: source_code, + }; + + console.log(">>> Got code ", job); + roomManager.broadcastToRoom(room_id, { + type: "code-execution-started", + }); + + // delete old timers + if (activeTimers.has(room_id)) { + clearTimeout(activeTimers.get(room_id)); + activeTimers.delete(room_id); } -}) + + const newTimer = setTimeout(() => { + console.log(">>> Job timeout set for ", room_id); + + activeTimers.delete(room_id); + + const timeoutMessage = { + type: "code-execution-result", + data: { + isError: true, + output: "Execution timed out. Please try again", + }, + }; + + roomManager.broadcastToRoom(room_id, timeoutMessage); + }, EXECUTION_TIMEOUT_MS); + + activeTimers.set(room_id, newTimer); + + // send job to MQ + await sendJob(job); + res.status(202).json({ + message: "Job accepted", + }); + } catch (error) { + console.error("Error during submission process:", error.message); + res.status(500).json({ + error: "Internal Server Error", + }); + } +}); /** * CALLBACK @@ -107,45 +93,44 @@ router.post("/submit-code", async (req, res) => { * get result and push to FE */ router.post("/execute-callback", async (req, res) => { - try { - // get result - const {room_id, isError, output} = req.body - - // delete the timeout or ignore if the job is timeout - if (activeTimers.has(room_id)) { - clearTimeout(activeTimers.get(room_id)) - activeTimers.delete(room_id) - } else { - console.log(`>>> Received STALE callback for ${room_id}`) - return res.status(200).json({message: 'Job already timeout'}) - } - - // send result to FE - const message = { - type: "code-execution-result", - data: { - isError: isError, - output: output - } - } - - roomManager.broadcastToRoom(room_id, message) - - console.log('>>> Callback, broadcasted to room') - console.log('room_id: ', room_id) - console.log('isError: ', isError) - console.log('output:', output) - - res.status(200).json({ - message: 'Got result and broadcasted to room' - }) - - } catch (error) { - console.error('Error during submission process:', error.message) - res.status(500).json({ - error: 'Internal Server Error' - }) + try { + // get result + const { room_id, isError, output } = req.body; + + // delete the timeout or ignore if the job is timeout + if (activeTimers.has(room_id)) { + clearTimeout(activeTimers.get(room_id)); + activeTimers.delete(room_id); + } else { + console.log(`>>> Received STALE callback for ${room_id}`); + return res.status(200).json({ message: "Job already timeout" }); } -}) -export default router + // send result to FE + const message = { + type: "code-execution-result", + data: { + isError: isError, + output: output, + }, + }; + + roomManager.broadcastToRoom(room_id, message); + + console.log(">>> Callback, broadcasted to room"); + console.log("room_id: ", room_id); + console.log("isError: ", isError); + console.log("output:", output); + + res.status(200).json({ + message: "Got result and broadcasted to room", + }); + } catch (error) { + console.error("Error during submission process:", error.message); + res.status(500).json({ + error: "Internal Server Error", + }); + } +}); + +export default router; diff --git a/backend/collaboration-service/src/server.js b/backend/collaboration-service/src/server.js index 00807fc04c..36b7dafd07 100644 --- a/backend/collaboration-service/src/server.js +++ b/backend/collaboration-service/src/server.js @@ -1,7 +1,7 @@ import db from "./db.js"; import httpServer from "./http/httpServer.js"; import webSocketServer from "./websocket/websocketServer.js"; -import {initRabbitMQ} from "./http/codeExecutionRoutes.js"; +import { initCodeExecution } from "./http/codeExecutionRoutes.js"; import { setupRoomCreationConsumer } from "./consumers/roomCreationConsumer.js"; async function startServer() { @@ -9,8 +9,8 @@ async function startServer() { // Step 1: Connect to MongoDB await db.connect(); - // Step 2: Connect to RabbitMQ - await initRabbitMQ(); + // Step 2: Connect to RabbitMQ for code execution + await initCodeExecution(); // Step 3: Start HTTP server await httpServer.start(); diff --git a/backend/execution-service/.env.example b/backend/execution-service/.env.example index 2e76322f7f..4fcef578b5 100644 --- a/backend/execution-service/.env.example +++ b/backend/execution-service/.env.example @@ -1,4 +1,4 @@ RABBITMQ_URL="amqp://user:password@rabbitmq" -QUEUE_NAME="execution_jobs" +QUEUE_NAME="job_execution_queue" CALLBACK_URL="http://collaboration-service:8004/api/v1/code/execute-callback" PISTON_URL="http://piston-api:2000/api/v2/execute" diff --git a/backend/execution-service/package.json b/backend/execution-service/package.json index 4dfda6e2ef..4f8115a048 100644 --- a/backend/execution-service/package.json +++ b/backend/execution-service/package.json @@ -2,10 +2,10 @@ "name": "execution-service", "version": "1.0.0", "description": "", - "main": "src/worker.js", + "main": "src/index.js", "scripts": { - "start": "node src/worker.js", - "dev:worker": "nodemon src/worker.js", + "start": "node src/index.js", + "dev:worker": "nodemon src/index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -13,6 +13,7 @@ "license": "ISC", "packageManager": "pnpm@10.17.1", "dependencies": { + "@google-cloud/pubsub": "^5.2.0", "amqplib": "^0.10.9", "axios": "^1.12.2" }, diff --git a/backend/execution-service/src/index.js b/backend/execution-service/src/index.js new file mode 100644 index 0000000000..3ad7959eaf --- /dev/null +++ b/backend/execution-service/src/index.js @@ -0,0 +1,33 @@ +const RabbitMQWorker = require("./rabbitmqWorker"); +const PubSubWorker = require("./pubsubWorker"); + +const useGcp = !!process.env.PUBSUB_PROJECT_ID; + +async function main() { + let worker; + + if (useGcp) { + console.log("Starting Pub/Sub worker..."); + worker = new PubSubWorker(); + } else { + console.log("Starting RabbitMQ worker..."); + worker = new RabbitMQWorker(); + } + + await worker.start(); + + // Graceful shutdown + process.on("SIGTERM", async () => { + console.log("SIGTERM signal received: closing worker"); + await worker.stop(); + process.exit(0); + }); + + process.on("SIGINT", async () => { + console.log("SIGINT signal received: closing worker"); + await worker.stop(); + process.exit(0); + }); +} + +main(); diff --git a/backend/execution-service/src/jobProcessor.js b/backend/execution-service/src/jobProcessor.js new file mode 100644 index 0000000000..a629b0a741 --- /dev/null +++ b/backend/execution-service/src/jobProcessor.js @@ -0,0 +1,147 @@ +const axios = require("axios"); + +const CALLBACK_URL = process.env.CALLBACK_URL; +const PISTON_URL = process.env.PISTON_URL; + +const MAX_RETRIES = 3; +const RETRY_DELAY_MS = 2000; +const AXIOS_CONNECTION_TIMEOUT_MS = 2000; +const PISTON_EXECUTION_TIMEOUT_MS = 40000; // 40s + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function callPistonAPI(language, source_code, timeout_ms) { + try { + const payload = { + language: language, + version: "*", + files: [ + { + content: source_code, + }, + ], + run_timeout: 10000, + compile_timeout: 30000, + }; + const response = await axios.post(PISTON_URL, payload, { + timeout: timeout_ms, + }); + return response.data; + } catch (error) { + throw error; + } +} + +async function resultCallback(result) { + for (let i = 1; i <= MAX_RETRIES; i++) { + try { + await axios.post(CALLBACK_URL, result); + console.log(">>> Sent callback: ", result.room_id); + return; + } catch (callbackError) { + if (i == MAX_RETRIES) { + // final error + throw callbackError; + } else { + const delay = i * RETRY_DELAY_MS; + await sleep(delay); + } + } + } +} + +/** + * + * @param {*} submit + * @returns result: {room_id, isError, output} + */ +async function processSubmission(submit) { + let result = {}; + result.room_id = submit.room_id; + + // first 2 time: check for connection + for (let i = 1; i <= MAX_RETRIES - 1; i++) { + try { + const response = await callPistonAPI( + submit.language, + submit.source_code, + AXIOS_CONNECTION_TIMEOUT_MS, + ); + } catch (error) { + if (error.response) { + const statusCode = error.response.status; + if (statusCode >= 400 && statusCode < 500) { + result.isError = true; + result.output = error.response.data.message; + console.log(">>> Piston api error: ", error.response.message); + console.log(">>> Finish job (error)", result); + return result; + } else { + // piston temp error + console.log(">>> Piston api temporary error: ", statusCode); + } + } else if (axios.isAxiosError(error) && error.code !== "ECONNREFUSED") { + // timeout / no connection + console.log(">>> Piston api doesn't respond\\n", error.message); + } else { + // ECONNREFUSED + console.log(">>> Piston api is dead (ECONNREFUSED)\\n", error.message); + } + + // next try + const delay = RETRY_DELAY_MS * i; + console.log(`>>> Wait ${delay}ms before reconnect to Piston api`); + await sleep(delay); + continue; + } + // if no error, quit try loop + break; + } + + // final try: timeout 40s + try { + const response = await callPistonAPI( + submit.language, + submit.source_code, + PISTON_EXECUTION_TIMEOUT_MS, + ); + + // set isError + if (response.run.status) { + // runtime error + result.isError = true; + // get readable message or output + result.output = response.run.message || response.run.output; + } else if (response.run.code !== 0) { + // runcode != 0 error + result.isError = true; + result.output = response.run.output; + } else { + // successfully run the code + result.isError = false; + result.output = response.run.output; + } + + console.log(">>> Finish job", result); + return result; + } catch (error) { + if (error.response) { + const statusCode = error.response.status; + if (statusCode >= 400 && statusCode < 500) { + result.isError = true; + result.output = error.response.data.message; + return result; + } + } + // failed (after 3 tries) + result.isError = true; + result.output = "Code execution service is unavailable. Please try again later."; + console.log(">>> Failed job ", result); + return result; + } +} + +module.exports = { + processSubmission, + resultCallback, +}; diff --git a/backend/execution-service/src/pubsubWorker.js b/backend/execution-service/src/pubsubWorker.js new file mode 100644 index 0000000000..ac301ece8c --- /dev/null +++ b/backend/execution-service/src/pubsubWorker.js @@ -0,0 +1,78 @@ +const { PubSub } = require("@google-cloud/pubsub"); +const { processSubmission, resultCallback } = require("./jobProcessor"); + +const JOB_EXECUTION_TOPIC = "job_execution_topic"; +const JOB_EXECUTION_SUBSCRIPTION = "job_execution_sub"; + +class PubSubWorker { + constructor() { + this.pubsub = new PubSub({ + projectId: process.env.PUBSUB_PROJECT_ID, + }); + this.subscription = null; + } + + async start() { + try { + const topic = this.pubsub.topic(JOB_EXECUTION_TOPIC); + + // Ensure topic exists + const [topicExists] = await topic.exists(); + if (!topicExists) { + await topic.create(); + console.log(`Created topic: ${JOB_EXECUTION_TOPIC}`); + } + + // Get or create subscription + this.subscription = topic.subscription(JOB_EXECUTION_SUBSCRIPTION); + const [subExists] = await this.subscription.exists(); + if (!subExists) { + [this.subscription] = await topic.createSubscription(JOB_EXECUTION_SUBSCRIPTION); + console.log(`Created subscription: ${JOB_EXECUTION_SUBSCRIPTION}`); + } + + console.log(">>> Consumer connected to Pub/Sub"); + + // Handle messages + const messageHandler = async (message) => { + try { + const job = JSON.parse(message.data.toString()); + console.log(">>> Got job: ", job.room_id); + + const result = await processSubmission(job); + + // callback + try { + await resultCallback(result); + } catch (callbackError) { + console.log(">>> Error when callback ", callbackError.message); + } finally { + message.ack(); + } + } catch (error) { + console.error("Error processing message:", error); + message.nack(); + } + }; + + this.subscription.on("message", messageHandler); + this.subscription.on("error", (error) => { + console.error("Subscription error:", error); + }); + } catch (error) { + console.log(">>> Worker's error: ", error); + process.exit(1); + } + } + + async stop() { + if (this.subscription) { + await this.subscription.close(); + } + if (this.pubsub) { + await this.pubsub.close(); + } + } +} + +module.exports = PubSubWorker; diff --git a/backend/execution-service/src/rabbitmqWorker.js b/backend/execution-service/src/rabbitmqWorker.js new file mode 100644 index 0000000000..6c93017318 --- /dev/null +++ b/backend/execution-service/src/rabbitmqWorker.js @@ -0,0 +1,61 @@ +const amqp = require("amqplib"); +const { processSubmission, resultCallback } = require("./jobProcessor"); + +const RABBITMQ_URL = process.env.RABBITMQ_URL; +const QUEUE_NAME = process.env.QUEUE_NAME || "job_execution_queue"; + +class RabbitMQWorker { + constructor() { + this.connection = null; + this.channel = null; + } + + async start() { + try { + this.connection = await amqp.connect(RABBITMQ_URL); + this.channel = await this.connection.createChannel(); + + await this.channel.assertQueue(QUEUE_NAME, { durable: true }); + + console.log(">>> Consumer connected to RabbitMQ"); + + this.channel.consume( + QUEUE_NAME, + async (msg) => { + if (msg != null) { + const job = JSON.parse(msg.content.toString()); + console.log(">>> Got job: ", job.room_id); + + const result = await processSubmission(job); + + // callback + try { + await resultCallback(result); + } catch (callbackError) { + console.log(">>> Error when callback ", callbackError.message); + } finally { + this.channel.ack(msg); + } + } + }, + { + noAck: false, + }, + ); + } catch (error) { + console.log(">>> Worker's error: ", error); + process.exit(1); + } + } + + async stop() { + if (this.channel) { + await this.channel.close(); + } + if (this.connection) { + await this.connection.close(); + } + } +} + +module.exports = RabbitMQWorker; diff --git a/backend/execution-service/src/worker.js b/backend/execution-service/src/worker.js index 72c7b14376..4e92ace414 100644 --- a/backend/execution-service/src/worker.js +++ b/backend/execution-service/src/worker.js @@ -1,175 +1,178 @@ -const aqmp = require('amqplib') -const axios = require('axios') - -const RABBITMQ_URL = process.env.RABBITMQ_URL -const QUEUE_NAME = process.env.QUEUE_NAME -const CALLBACK_URL = process.env.CALLBACK_URL -const PISTON_URL = process.env.PISTON_URL - -const MAX_RETRIES = 3 -const RETRY_DELAY_MS = 2000 -const AXIOS_CONNECTION_TIMEOUT_MS = 2000 -const PISTON_EXECUTION_TIMEOUT_MS = 40000 // 40s - -const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); - -async function callPistonAPI(language, source_code, timeout_ms) { - try { - const pistonURL = "http://piston:2000/api/v2/execute" - const payload = { - language: language, - version: "*", - files: [ - { - content: source_code - } - ], - run_timeout: 10000, - compile_timeout: 30000 - } - const response = await axios.post(PISTON_URL, payload, { - timeout: timeout_ms - }) - return response.data - } catch (error) { - throw error - } -} - -async function resultCallback(result) { - for (let i = 1; i <= MAX_RETRIES; i++) { - try { - await axios.post(CALLBACK_URL, result) - console.log('>>> Sent callback: ', result.room_id) - return - } catch (callbackError) { - if (i == MAX_RETRIES) { // final error - throw callbackError - } else { - const delay = i * RETRY_DELAY_MS - await sleep(delay) - } - } - } -} - /** - * - * @param {*} submit - * @returns result: {room_id, isError, output} + * @deprecated This file is refactored into index.js */ -async function processSubmission(submit) { - let result = {} - result.room_id = submit.room_id - - // first 2 time: check for connection - for (let i = 1; i <= MAX_RETRIES - 1; i++) { - try { - const response = await callPistonAPI(submit.language, submit.source_code, AXIOS_CONNECTION_TIMEOUT_MS) - } catch (error) { - if (error.response) { - const statusCode = error.response.status - if (statusCode >= 400 && statusCode < 500) { - result.isError = true - result.output = error.response.data.message - console.log(">>> Piston api error: ", error.response.message) - console.log(">>> Finish job (error)", result) - return result - } else { - // piston temp error - console.log(">>> Piston api temporary error: ", statusCode) - } - } else if (axios.isAxiosError(error) && error.code !== 'ECONNREFUSED') { - // timeout / no connection - console.log(">>> Piston api doesn't respond\n", error.message) - } else { - // ECONNREFUSED - console.log(">>> Piston api is dead (ECONNREFUSED)\n", error.message) - } - - // next try - const delay = RETRY_DELAY_MS * i; - console.log(`>>> Wait ${delay}ms before reconnect to Piston api`) - await sleep(delay) - continue - } - // if no error, quit try loop - break - } - - // final try: timeout 40s - try { - const response = await callPistonAPI(submit.language, submit.source_code, PISTON_EXECUTION_TIMEOUT_MS) - - // set isError - if (response.run.status) { // runtime error - result.isError = true - // get readable message or output - result.output = response.run.message || response.run.output - } else if (response.run.code !== 0) { // runcode != 0 error - result.isError = true - result.output = response.run.output - } else { // successfully run the code - result.isError = false - result.output = response.run.output - } - - - console.log(">>> Finish job", result) - return result - } catch (error) { - if (error.response) { - const statusCode = error.response.status - if (statusCode >= 400 && statusCode < 500) { - result.isError = true - result.output = error.response.data.message - return result - } - } - // failed (after 3 tries) - result.isError = true - result.output = "Code execution service is unavailable. Please try again later." - console.log('>>> Failed job ', result) - return result - } -} - -async function startWorker() { - try { - const connection = await aqmp.connect(RABBITMQ_URL) - const channel = await connection.createChannel() - - await channel.assertQueue(QUEUE_NAME, { durable: true}) - - console.log(">>> Consumer connected RabbitMQ") - - channel.consume(QUEUE_NAME, async (msg) => { - if (msg != null) { - const job = JSON.parse(msg.content.toString()) - console.log('>>> Got job: ', job.room_id) - - const result = await processSubmission(job) - - // callback - try { - await resultCallback(result) - } catch (callbackError) { - console.log(">>> Error when callback ", callbackError.message) - } finally { - channel.ack(msg) - } - } - }, { - noAck: false - }) - } catch (error) { - console.log(">>> Worker's error: ", error) - process.exit(1) - } -} - -async function main() { - startWorker() -} - -main() + +// const aqmp = require('amqplib') +// const axios = require('axios') + +// const RABBITMQ_URL = process.env.RABBITMQ_URL +// const QUEUE_NAME = process.env.QUEUE_NAME +// const CALLBACK_URL = process.env.CALLBACK_URL +// const PISTON_URL = process.env.PISTON_URL + +// const MAX_RETRIES = 3 +// const RETRY_DELAY_MS = 2000 +// const AXIOS_CONNECTION_TIMEOUT_MS = 2000 +// const PISTON_EXECUTION_TIMEOUT_MS = 40000 // 40s + +// const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +// async function callPistonAPI(language, source_code, timeout_ms) { +// try { +// const pistonURL = "http://piston:2000/api/v2/execute" +// const payload = { +// language: language, +// version: "*", +// files: [ +// { +// content: source_code +// } +// ], +// run_timeout: 10000, +// compile_timeout: 30000 +// } +// const response = await axios.post(PISTON_URL, payload, { +// timeout: timeout_ms +// }) +// return response.data +// } catch (error) { +// throw error +// } +// } + +// async function resultCallback(result) { +// for (let i = 1; i <= MAX_RETRIES; i++) { +// try { +// await axios.post(CALLBACK_URL, result) +// console.log('>>> Sent callback: ', result.room_id) +// return +// } catch (callbackError) { +// if (i == MAX_RETRIES) { // final error +// throw callbackError +// } else { +// const delay = i * RETRY_DELAY_MS +// await sleep(delay) +// } +// } +// } +// } + +// /** +// * +// * @param {*} submit +// * @returns result: {room_id, isError, output} +// */ +// async function processSubmission(submit) { +// let result = {} +// result.room_id = submit.room_id + +// // first 2 time: check for connection +// for (let i = 1; i <= MAX_RETRIES - 1; i++) { +// try { +// const response = await callPistonAPI(submit.language, submit.source_code, AXIOS_CONNECTION_TIMEOUT_MS) +// } catch (error) { +// if (error.response) { +// const statusCode = error.response.status +// if (statusCode >= 400 && statusCode < 500) { +// result.isError = true +// result.output = error.response.data.message +// console.log(">>> Piston api error: ", error.response.message) +// console.log(">>> Finish job (error)", result) +// return result +// } else { +// // piston temp error +// console.log(">>> Piston api temporary error: ", statusCode) +// } +// } else if (axios.isAxiosError(error) && error.code !== 'ECONNREFUSED') { +// // timeout / no connection +// console.log(">>> Piston api doesn't respond\n", error.message) +// } else { +// // ECONNREFUSED +// console.log(">>> Piston api is dead (ECONNREFUSED)\n", error.message) +// } + +// // next try +// const delay = RETRY_DELAY_MS * i; +// console.log(`>>> Wait ${delay}ms before reconnect to Piston api`) +// await sleep(delay) +// continue +// } +// // if no error, quit try loop +// break +// } + +// // final try: timeout 40s +// try { +// const response = await callPistonAPI(submit.language, submit.source_code, PISTON_EXECUTION_TIMEOUT_MS) + +// // set isError +// if (response.run.status) { // runtime error +// result.isError = true +// // get readable message or output +// result.output = response.run.message || response.run.output +// } else if (response.run.code !== 0) { // runcode != 0 error +// result.isError = true +// result.output = response.run.output +// } else { // successfully run the code +// result.isError = false +// result.output = response.run.output +// } + +// console.log(">>> Finish job", result) +// return result +// } catch (error) { +// if (error.response) { +// const statusCode = error.response.status +// if (statusCode >= 400 && statusCode < 500) { +// result.isError = true +// result.output = error.response.data.message +// return result +// } +// } +// // failed (after 3 tries) +// result.isError = true +// result.output = "Code execution service is unavailable. Please try again later." +// console.log('>>> Failed job ', result) +// return result +// } +// } + +// async function startWorker() { +// try { +// const connection = await aqmp.connect(RABBITMQ_URL) +// const channel = await connection.createChannel() + +// await channel.assertQueue(QUEUE_NAME, { durable: true}) + +// console.log(">>> Consumer connected RabbitMQ") + +// channel.consume(QUEUE_NAME, async (msg) => { +// if (msg != null) { +// const job = JSON.parse(msg.content.toString()) +// console.log('>>> Got job: ', job.room_id) + +// const result = await processSubmission(job) + +// // callback +// try { +// await resultCallback(result) +// } catch (callbackError) { +// console.log(">>> Error when callback ", callbackError.message) +// } finally { +// channel.ack(msg) +// } +// } +// }, { +// noAck: false +// }) +// } catch (error) { +// console.log(">>> Worker's error: ", error) +// process.exit(1) +// } +// } + +// async function main() { +// startWorker() +// } + +// main() diff --git a/docker-compose.yml b/docker-compose.yml index 1eafd7974f..97640f8f42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,7 +164,7 @@ services: - ./backend/execution-service/.env environment: - RABBITMQ_URL=amqp://user:password@rabbitmq - - QUEUE_NAME=execution_jobs + - QUEUE_NAME=job_execution_queue - CALLBACK_URL=http://collaboration-service:8004/api/v1/code/execute-callback - PISTON_URL=http://piston-api:2000/api/v2/execute depends_on: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a12c4f182f..5fb8399e27 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: backend/execution-service: dependencies: + '@google-cloud/pubsub': + specifier: ^5.2.0 + version: 5.2.0 amqplib: specifier: ^0.10.9 version: 0.10.9 From ce9a4aaeb1cca326f8e15d5bbf9bcffb8bf87d3e Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Wed, 12 Nov 2025 03:26:07 +0800 Subject: [PATCH 11/12] feat(terraform): deploy video call service --- backend/video-call-service/.env.example | 4 +- frontend/Dockerfile.frontend | 2 + frontend/src/lib/api-config.ts | 4 ++ frontend/src/stores/collaboration-store.ts | 4 +- scripts/build-and-push.sh | 3 + terraform/loadbalancer.tf | 36 +++++++++++- terraform/main.tf | 65 ++++++++++++++++++++++ terraform/outputs.tf | 5 ++ 8 files changed, 117 insertions(+), 6 deletions(-) diff --git a/backend/video-call-service/.env.example b/backend/video-call-service/.env.example index a85707b129..3eb3ad66cf 100644 --- a/backend/video-call-service/.env.example +++ b/backend/video-call-service/.env.example @@ -1,3 +1,3 @@ -APP_ID=997489f851fa4186a4b9c65bab955777 -APP_CERTIFICATE=63c44822bae140a3ade7153d209233f5 +APP_ID= +APP_CERTIFICATE= PORT=8011 \ No newline at end of file diff --git a/frontend/Dockerfile.frontend b/frontend/Dockerfile.frontend index 712761b85f..bc9f38eec8 100644 --- a/frontend/Dockerfile.frontend +++ b/frontend/Dockerfile.frontend @@ -25,11 +25,13 @@ ARG USER_SERVICE_URL ARG MATCHING_SERVICE_URL ARG QUESTION_SERVICE_URL ARG COLLABORATION_SERVICE_URL +ARG VIDEO_CALL_SERVICE_URL ENV NEXT_PUBLIC_USER_SERVICE_URL=$USER_SERVICE_URL \ NEXT_PUBLIC_MATCHING_SERVICE_URL=$MATCHING_SERVICE_URL \ NEXT_PUBLIC_QUESTION_SERVICE_URL=$QUESTION_SERVICE_URL \ NEXT_PUBLIC_COLLABORATION_SERVICE_URL=$COLLABORATION_SERVICE_URL \ + NEXT_PUBLIC_VIDEO_CALL_SERVICE_URL=$VIDEO_CALL_SERVICE_URL \ NEXT_TELEMETRY_DISABLED=1 \ NEXT_DISABLE_SOURCEMAPS=1 diff --git a/frontend/src/lib/api-config.ts b/frontend/src/lib/api-config.ts index 8cc14c8be5..e65a8ff6af 100644 --- a/frontend/src/lib/api-config.ts +++ b/frontend/src/lib/api-config.ts @@ -11,9 +11,13 @@ export const apiConfig = { collaborationService: { baseURL: process.env.NEXT_PUBLIC_COLLABORATION_SERVICE_URL || "http://localhost:8004", }, + videoCallService: { + baseURL: process.env.NEXT_PUBLIC_VIDEO_CALL_SERVICE_URL || "http://localhost:8011", + }, } as const; export const getUserServiceURL = () => apiConfig.userService.baseURL; export const getMatchingServiceURL = () => apiConfig.matchingService.baseURL; export const getQuestionServiceURL = () => apiConfig.questionService.baseURL; export const getCollaborationURL = () => apiConfig.collaborationService.baseURL; +export const getVideoCallServiceURL = () => apiConfig.videoCallService.baseURL; diff --git a/frontend/src/stores/collaboration-store.ts b/frontend/src/stores/collaboration-store.ts index 9db9a3b39c..e13f7691ac 100644 --- a/frontend/src/stores/collaboration-store.ts +++ b/frontend/src/stores/collaboration-store.ts @@ -4,11 +4,11 @@ import axios, { AxiosError } from "axios"; import { toast } from "react-toastify"; import { ProgrammingLanguage } from "@/utils/enums"; import { collaborationAPI, RoomDetails, questionAPI, Question } from "@/lib/api-client"; -import { getCollaborationURL } from "@/lib/api-config"; +import { getCollaborationURL, getVideoCallServiceURL } from "@/lib/api-config"; let executionTimer: NodeJS.Timeout | null = null; const EXECUTION_TIMEOUT_MS = 60000; // 60s -const VIDEO_CALL_URL = "http://localhost:8011/v1/video/"; +const VIDEO_CALL_URL = getVideoCallServiceURL() + "/v1/video/"; const MAX_RETRIES = 3; const RETRY_DELAY_MS = 1500; diff --git a/scripts/build-and-push.sh b/scripts/build-and-push.sh index 0df2e0faf6..4e997f5347 100755 --- a/scripts/build-and-push.sh +++ b/scripts/build-and-push.sh @@ -15,6 +15,7 @@ USER_SERVICE_URL="https://${DOMAIN}/api/user" MATCHING_SERVICE_URL="https://${DOMAIN}/api/matching" QUESTION_SERVICE_URL="https://${DOMAIN}/api/question" COLLABORATION_SERVICE_URL="https://${DOMAIN}/api/collaboration" +VIDEO_CALL_SERVICE_URL="https://${DOMAIN}/api/video-call" echo "🔨 Building and pushing Docker images to ${REGISTRY}" @@ -30,6 +31,7 @@ SERVICES=( "question-service:backend/question-service:Dockerfile.question" "collaboration-service:backend/collaboration-service:Dockerfile.collaboration" "execution-service:backend/execution-service:Dockerfile.execution" + "video-call-service:backend/video-call-service:Dockerfile.video" "frontend:frontend:Dockerfile.frontend" ) @@ -57,6 +59,7 @@ for SERVICE_CONFIG in "${SERVICES[@]}"; do --build-arg MATCHING_SERVICE_URL="$MATCHING_SERVICE_URL" \ --build-arg QUESTION_SERVICE_URL="$QUESTION_SERVICE_URL" \ --build-arg COLLABORATION_SERVICE_URL="$COLLABORATION_SERVICE_URL" \ + --build-arg VIDEO_CALL_SERVICE_URL="$VIDEO_CALL_SERVICE_URL" \ -f "${SERVICE_PATH}/${DOCKERFILE}" -t "${REGISTRY}/${SERVICE_NAME}:latest" --load "${SERVICE_PATH}" else docker buildx build --platform linux/amd64 \ diff --git a/terraform/loadbalancer.tf b/terraform/loadbalancer.tf index b1897cd33f..31313b61c4 100644 --- a/terraform/loadbalancer.tf +++ b/terraform/loadbalancer.tf @@ -40,6 +40,15 @@ resource "google_compute_region_network_endpoint_group" "collab_service_neg" { } } +resource "google_compute_region_network_endpoint_group" "video_call_service_neg" { + name = "video-call-service-neg" + network_endpoint_type = "SERVERLESS" + region = var.region + cloud_run { + service = google_cloud_run_v2_service.video_call_service.name + } +} + resource "google_compute_region_network_endpoint_group" "frontend_neg" { name = "frontend-neg" network_endpoint_type = "SERVERLESS" @@ -90,6 +99,16 @@ resource "google_compute_backend_service" "collab_service_backend" { } } +resource "google_compute_backend_service" "video_call_service_backend" { + name = "video-call-service-backend" + protocol = "HTTP" + port_name = "http" + load_balancing_scheme = "EXTERNAL_MANAGED" + backend { + group = google_compute_region_network_endpoint_group.video_call_service_neg.id + } +} + resource "google_compute_backend_service" "frontend_backend" { name = "frontend-backend" protocol = "HTTP" @@ -143,7 +162,7 @@ resource "google_compute_url_map" "api_url_map" { } } } - + route_rules { priority = 3 match_rules { @@ -156,7 +175,7 @@ resource "google_compute_url_map" "api_url_map" { } } } - + route_rules { priority = 4 match_rules { @@ -169,6 +188,19 @@ resource "google_compute_url_map" "api_url_map" { } } } + + route_rules { + priority = 5 + match_rules { + prefix_match = "/api/video-call/" + } + service = google_compute_backend_service.video_call_service_backend.id + route_action { + url_rewrite { + path_prefix_rewrite = "/" + } + } + } } } diff --git a/terraform/main.tf b/terraform/main.tf index d88f3914e8..534a2904a7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -465,6 +465,64 @@ resource "google_cloud_run_v2_service" "collaboration_service" { depends_on = [google_project_service.apis] } +# Cloud Run Service - Video Call Service +resource "google_cloud_run_v2_service" "video_call_service" { + name = "video-call-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/video-call-service:latest" + + ports { + container_port = 8011 + } + + env { + name = "APP_ID" + value_source { + secret_key_ref { + secret = "video_call_service_app_id" + version = "latest" + } + } + } + + env { + name = "APP_CERTIFICATE" + value_source { + secret_key_ref { + secret = "video_call_service_app_certificate" + version = "latest" + } + } + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + # Cloud Run Service - Frontend resource "google_cloud_run_v2_service" "frontend" { name = "frontend" @@ -532,6 +590,13 @@ resource "google_cloud_run_service_iam_member" "collaboration_service_public" { member = "allUsers" } +resource "google_cloud_run_service_iam_member" "video_call_service_public" { + service = google_cloud_run_v2_service.video_call_service.name + location = google_cloud_run_v2_service.video_call_service.location + role = "roles/run.invoker" + member = "allUsers" +} + resource "google_cloud_run_service_iam_member" "frontend_public" { service = google_cloud_run_v2_service.frontend.name location = google_cloud_run_v2_service.frontend.location diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 4807d7acd7..5d8606a096 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -28,6 +28,11 @@ output "collaboration_service_url" { value = google_cloud_run_v2_service.collaboration_service.uri } +output "video_call_service_url" { + description = "Video call service URL" + value = google_cloud_run_v2_service.video_call_service.uri +} + output "redis_host" { description = "Redis host address" value = google_redis_instance.redis.host From a53ba730ed95172eeac9537a25c5c9c6e1a11705 Mon Sep 17 00:00:00 2001 From: Nguyen Cao Duy Date: Wed, 12 Nov 2025 09:45:03 +0800 Subject: [PATCH 12/12] feat(terraform): deploy execution service --- backend/execution-service/src/index.js | 20 +++++++ terraform/main.tf | 80 +++++++++++++++++++++++++- terraform/outputs.tf | 5 ++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/backend/execution-service/src/index.js b/backend/execution-service/src/index.js index 3ad7959eaf..6eb4da4b11 100644 --- a/backend/execution-service/src/index.js +++ b/backend/execution-service/src/index.js @@ -1,7 +1,9 @@ +const http = require("http"); const RabbitMQWorker = require("./rabbitmqWorker"); const PubSubWorker = require("./pubsubWorker"); const useGcp = !!process.env.PUBSUB_PROJECT_ID; +const PORT = process.env.PORT || 8080; async function main() { let worker; @@ -16,16 +18,34 @@ async function main() { await worker.start(); + let server = null; + if (useGcp) { + // Create a minimal HTTP server for GCP Cloud Run health checks + server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + data: 'Hello World!', + })); + }); + server.listen(PORT); + } + // Graceful shutdown process.on("SIGTERM", async () => { console.log("SIGTERM signal received: closing worker"); await worker.stop(); + if (server) { + server.close(); + } process.exit(0); }); process.on("SIGINT", async () => { console.log("SIGINT signal received: closing worker"); await worker.stop(); + if (server) { + server.close(); + } process.exit(0); }); } diff --git a/terraform/main.tf b/terraform/main.tf index 534a2904a7..cd5afc0000 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -75,7 +75,7 @@ resource "google_redis_instance" "redis" { depends_on = [google_project_service.apis] } -# Pub/Sub Topics (Kafka replacement) +# Pub/Sub Topics (Kafka/RabbitMQ replacement) resource "google_pubsub_topic" "match_topic" { name = "match_topic" @@ -100,7 +100,13 @@ resource "google_pubsub_topic" "room_created_topic" { depends_on = [google_project_service.apis] } -# Pub/Sub Subscriptions (Kafka replacement) +resource "google_pubsub_topic" "job_execution_topic" { + name = "job_execution_topic" + + depends_on = [google_project_service.apis] +} + +# Pub/Sub Subscriptions (Kafka/RabbitMQ replacement) resource "google_pubsub_subscription" "match_sub" { name = "match_sub" topic = google_pubsub_topic.match_topic.name @@ -149,6 +155,18 @@ resource "google_pubsub_subscription" "room_created_sub" { } } +resource "google_pubsub_subscription" "job_execution_sub" { + name = "job_execution_sub" + topic = google_pubsub_topic.job_execution_topic.name + + ack_deadline_seconds = 20 + + retry_policy { + minimum_backoff = "10s" + maximum_backoff = "600s" + } +} + # Artifact Registry for Docker images resource "google_artifact_registry_repository" "docker_repo" { location = var.region @@ -465,6 +483,57 @@ resource "google_cloud_run_v2_service" "collaboration_service" { depends_on = [google_project_service.apis] } +resource "google_cloud_run_v2_service" "execution_service" { + name = "execution-service" + location = var.region + deletion_protection = false + + template { + service_account = google_service_account.service_account.email + + containers { + image = "${var.region}-docker.pkg.dev/${var.project_id}/docker-repo/execution-service:latest" + + ports { + container_port = 8080 + } + + env { + name = "CALLBACK_URL" + value = "${google_cloud_run_v2_service.collaboration_service.uri}/api/v1/code/execute-callback" + } + + env { + name = "PISTON_URL" + value = "https://emkc.org/api/v2/piston/execute" + } + env { + name = "PUBSUB_PROJECT_ID" + value = var.project_id + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + } + + scaling { + min_instance_count = 1 + max_instance_count = 5 + } + + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } + + depends_on = [google_project_service.apis] +} + # Cloud Run Service - Video Call Service resource "google_cloud_run_v2_service" "video_call_service" { name = "video-call-service" @@ -590,6 +659,13 @@ resource "google_cloud_run_service_iam_member" "collaboration_service_public" { member = "allUsers" } +resource "google_cloud_run_service_iam_member" "execution_service_public" { + service = google_cloud_run_v2_service.execution_service.name + location = google_cloud_run_v2_service.execution_service.location + role = "roles/run.invoker" + member = "allUsers" +} + resource "google_cloud_run_service_iam_member" "video_call_service_public" { service = google_cloud_run_v2_service.video_call_service.name location = google_cloud_run_v2_service.video_call_service.location diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 5d8606a096..1b0a637a94 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -28,6 +28,11 @@ output "collaboration_service_url" { value = google_cloud_run_v2_service.collaboration_service.uri } +output "execution_service_url" { + description = "Execution service URL" + value = google_cloud_run_v2_service.execution_service.uri +} + output "video_call_service_url" { description = "Video call service URL" value = google_cloud_run_v2_service.video_call_service.uri