Merge pull request #5 from godotengine/compose-db-retry

compose-db: Add retry logic like in interactive changelog

Author: akien-mga

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ name: Continuous integration
 on:
   push:
     branches: [ master ]
+  pull_request:
   schedule:
     # Run every 15 minutes of every hour, starting at 5 minutes (5m, 20m, 35m, 50m).
     # The slight offset is there to try and avoid the high load times.
@@ -52,6 +53,7 @@ jobs:
         path: out
 
     - name: Deploy to GitHub Pages 🚀
+      if: github.event_name != 'pull_request'
       uses: JamesIves/github-pages-deploy-action@v4
       with:
         branch: gh-pages

compose-db.js

@@ -8,6 +8,8 @@ const ExitCodes = {
 };
 
 const PULLS_PER_PAGE = 100;
+const API_DELAY_MSEC = 2500;
+const API_MAX_RETRIES = 10;
 const API_RATE_LIMIT = `
   rateLimit {
     limit
@@ -58,8 +60,12 @@ class DataFetcher {
            console.log(`    [${item.type}] ${item.message}`);
         });
     }
-    
-    async fetchGithub(query) {
+
+    async delay(msec) {
+        return new Promise(resolve => setTimeout(resolve, msec));
+    }
+
+    async fetchGithub(query, retries = 0) {
         const init = {};
         init.method = "POST";
         init.headers = {};
@@ -68,13 +74,42 @@ class DataFetcher {
             init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`;
         } else if (process.env.GITHUB_TOKEN) {
             init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`;
+        } else {
+            console.error("    Unable to find environment variable: `GRAPHQL_TOKEN`. Did you forget to set it in your local environment or a root `.env` file?");
+            process.exitCode = buildCommon.ExitCodes.ParseFailure;
+            return [null, null];
         }
-    
+
         init.body = JSON.stringify({
             query,
         });
-    
-        return await fetch("https://api.github.com/graphql", init);
+
+        let res = await fetch("https://api.github.com/graphql", init);
+        let attempt = 0;
+
+        while (true) {
+            if (attempt > retries) {
+                return [res, null];
+            }
+
+            if (res.status === 200) {
+                try {
+                    const json = await res.json()
+                    return [res, json];
+                }
+                catch (err) {
+                    console.log(`    Failed due to invalid response body, retrying (${attempt}/${retries})...`);
+                }
+            }
+            else {
+                console.log(`    Failed with status ${res.status}, retrying (${attempt}/${retries})...`);
+            }
+
+            // GitHub API is flaky, so we add an extra delay to let it calm down a bit.
+            await this.delay(API_DELAY_MSEC);
+            attempt += 1;
+            res = await fetch("https://api.github.com/graphql", init);
+        }
     }
 
     async fetchGithubRest(query) {
@@ -86,8 +121,12 @@ class DataFetcher {
             init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`;
         } else if (process.env.GITHUB_TOKEN) {
             init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`;
+        } else {
+            console.error("    Unable to find environment variable: `GRAPHQL_TOKEN`. Did you forget to set it in your local environment or a root `.env` file?");
+            process.exitCode = buildCommon.ExitCodes.ParseFailure;
+            return null;
         }
-    
+
         return await fetch(`${this.api_rest_path}${query}`, init);
     }
 
@@ -99,14 +138,13 @@ class DataFetcher {
             }
             `;
 
-            const res = await this.fetchGithub(query);
-            if (res.status !== 200) {
+            const [res, data] = await this.fetchGithub(query);
+            if (res.status !== 200 || data == null) {
                 this._handleResponseErrors(this.api_repository_id, res);
                 process.exitCode = ExitCodes.RequestFailure;
                 return;
             }
 
-            const data = await res.json();
             await this._logResponse(data, "_rate_limit");
             this._handleDataErrors(data);
 
@@ -191,14 +229,13 @@ class DataFetcher {
             }
             console.log(`    Requesting page ${page_text} of pull request data (${after_text}).`);
 
-            const res = await this.fetchGithub(query);
-            if (res.status !== 200) {
+            const [res, data] = await this.fetchGithub(query, API_MAX_RETRIES);
+            if (res.status !== 200 || data == null) {
                 this._handleResponseErrors(this.api_repository_id, res);
                 process.exitCode = ExitCodes.RequestFailure;
                 return [];
             }
 
-            const data = await res.json();
             await this._logResponse(data, `data_page_${page}`);
             this._handleDataErrors(data);
 

package-lock.json

@@ -15,7 +15,7 @@
     "dompurify": "^2.0.7",
     "lit-element": "^2.2.1",
     "marked": "^0.7.0",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^2.7.0",
     "posthtml": "^0.12.0",
     "rollup": "^1.24.0",
     "rollup-plugin-babel": "^4.3.3",