Add dry runs to CI

jablko · jablko · commit ac83bde84ee4 · 2022-06-16T08:17:31.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -36,6 +36,21 @@ jobs:
       run: yarn lint
     - name: test
       run: yarn test
+  publish-packages-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/publish-packages.yml
+    with:
+      dry-run: true
+  publish-registry-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/publish-registry.yml
+    with:
+      dry-run: true
+  retag-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/retag.yml
+    with:
+      dry-run: true
   publish_alpha:
     name: publish alpha release
     runs-on: ubuntu-latest
diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml
@@ -0,0 +1,41 @@
+name: Publish packages
+on:
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  publish-packages:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Parse declarations
+        run: yarn workspace @definitelytyped/publisher parse
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Calculate versions
+        run: yarn workspace @definitelytyped/publisher calculate-versions
+      - name: Generate packages
+        run: yarn workspace @definitelytyped/publisher generate
+      - name: Publish packages${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn workspace @definitelytyped/publisher publish-packages${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/
diff --git a/.github/workflows/publish-registry.yml b/.github/workflows/publish-registry.yml
@@ -0,0 +1,36 @@
+name: Publish registry
+on:
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  publish-registry:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Parse declarations
+        run: yarn workspace @definitelytyped/publisher parse
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Publish registry${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn workspace @definitelytyped/publisher publish-registry${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/
diff --git a/.github/workflows/retag.yml b/.github/workflows/retag.yml
@@ -0,0 +1,38 @@
+name: Update npm tags
+on:
+  schedule:
+    # https://crontab.guru/#0_0_*_*_0
+    - cron: 0 0 * * 0
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  retag:
+    if: github.event_name != 'schedule' || github.repository == 'microsoft/DefinitelyTyped-tools'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Update npm tags${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn retag${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/
diff --git a/.github/workflows/update-ts-version-tags.yml b/.github/workflows/update-ts-version-tags.yml
diff --git a/package.json b/package.json
@@ -11,7 +11,7 @@
     "format": "prettier --write 'packages/**/*.ts'",
     "test": "jest",
     "build": "tsc -b .",
-    "retag": "node packages/retag/dist/retag.js"
+    "retag": "node --require source-map-support/register packages/retag/"
   },
   "devDependencies": {
     "@types/jest": "^25.1.3",
diff --git a/packages/publisher/src/calculate-versions.ts b/packages/publisher/src/calculate-versions.ts
@@ -1,4 +1,5 @@
-import { defaultLocalOptions } from "./lib/common";
+import * as process from "process";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { ChangedPackages, ChangedPackagesJson, ChangedTypingJson, versionsFilename } from "./lib/versions";
 import { getDefinitelyTyped, AllPackages, NotNeededPackage, writeDataFile } from "@definitelytyped/definitions-parser";
 import {
@@ -13,12 +14,15 @@ import {
   CachedNpmInfoClient,
 } from "@definitelytyped/utils";
 import { fetchTypesPackageVersionInfo } from "@definitelytyped/retag";
-import { cacheDirPath } from "./lib/settings";
 
 if (!module.parent) {
   const log = loggerWithErrors()[0];
   logUncaughtErrors(async () =>
-    calculateVersions(await getDefinitelyTyped(defaultLocalOptions, log), new UncachedNpmInfoClient(), log)
+    calculateVersions(
+      await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log),
+      new UncachedNpmInfoClient(),
+      log
+    )
   );
 }
 
@@ -28,15 +32,11 @@ export default async function calculateVersions(
   log: LoggerWithErrors
 ): Promise<ChangedPackages> {
   log.info("=== Calculating versions ===");
-  return withNpmCache(
-    uncachedClient,
-    async (client) => {
-      log.info("* Reading packages...");
-      const packages = await AllPackages.read(dt);
-      return computeAndSaveChangedPackages(packages, log, client);
-    },
-    cacheDirPath
-  );
+  return withNpmCache(uncachedClient, async (client) => {
+    log.info("* Reading packages...");
+    const packages = await AllPackages.read(dt);
+    return computeAndSaveChangedPackages(packages, log, client);
+  });
 }
 
 async function computeAndSaveChangedPackages(
diff --git a/packages/publisher/src/generate-packages.ts b/packages/publisher/src/generate-packages.ts
@@ -2,10 +2,11 @@ import { makeTypesVersionsForPackageJson } from "@definitelytyped/header-parser"
 import assert = require("assert");
 import { emptyDir, mkdir, mkdirp, readFileSync } from "fs-extra";
 import path = require("path");
+import * as process from "process";
 import yargs = require("yargs");
 
-import { defaultLocalOptions } from "./lib/common";
-import { outputDirPath, sourceBranch, cacheDirPath } from "./lib/settings";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
+import { outputDirPath, sourceBranch } from "./lib/settings";
 import {
   assertNever,
   joinPaths,
@@ -44,7 +45,7 @@ if (!module.parent) {
   const tgz = !!yargs.argv.tgz;
   logUncaughtErrors(async () => {
     const log = loggerWithErrors()[0];
-    const dt = await getDefinitelyTyped(defaultLocalOptions, log);
+    const dt = await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log);
     const allPackages = await AllPackages.read(dt);
     await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz);
   });
@@ -70,16 +71,12 @@ export default async function generatePackages(
     log(` * ${pkg.desc}`);
   }
   log("## Generating deprecated packages");
-  await withNpmCache(
-    new UncachedNpmInfoClient(),
-    async (client) => {
-      for (const pkg of changedPackages.changedNotNeededPackages) {
-        log(` * ${pkg.libraryName}`);
-        await generateNotNeededPackage(pkg, client, log);
-      }
-    },
-    cacheDirPath
-  );
+  await withNpmCache(new UncachedNpmInfoClient(), async (client) => {
+    for (const pkg of changedPackages.changedNotNeededPackages) {
+      log(` * ${pkg.libraryName}`);
+      await generateNotNeededPackage(pkg, client, log);
+    }
+  });
   await writeLog("package-generator.md", logResult());
 }
 async function generateTypingPackage(
diff --git a/packages/publisher/src/lib/settings.ts b/packages/publisher/src/lib/settings.ts
@@ -4,7 +4,6 @@ import { toS } from "hh-mm-ss";
 const hostJson = require("../../host.json");
 const root = joinPaths(__dirname, "..", "..");
 const storageDirPath = process.env.STORAGE_DIR || root;
-export const cacheDirPath = joinPaths(storageDirPath, "cache");
 export const outputDirPath = joinPaths(storageDirPath, "output");
 export const validateOutputPath = joinPaths(storageDirPath, "validateOutput");
 export const logDir = joinPaths(storageDirPath, "logs");
diff --git a/packages/publisher/src/parse-definitions.ts b/packages/publisher/src/parse-definitions.ts
@@ -1,7 +1,8 @@
 import os from "os";
+import * as process from "process";
 import yargs from "yargs";
 import { logUncaughtErrors, loggerWithErrors, assertDefined, FS } from "@definitelytyped/utils";
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { getTypingInfo } from "@definitelytyped/definitions-parser/dist/lib/definition-parser";
 import {
   getDefinitelyTyped,
@@ -16,7 +17,7 @@ if (!module.parent) {
     nProcesses: { type: "number" },
   }).argv;
 
-  const options = defaultLocalOptions;
+  const options = process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions;
   logUncaughtErrors(async () => {
     const log = loggerWithErrors()[0];
     const dt = await getDefinitelyTyped(options, log);
diff --git a/packages/publisher/src/publish-packages.ts b/packages/publisher/src/publish-packages.ts
@@ -1,7 +1,8 @@
+import * as process from "process";
 import applicationinsights = require("applicationinsights");
 import * as yargs from "yargs";
 
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { deprecateNotNeededPackage, publishNotNeededPackage, publishTypingsPackage } from "./lib/package-publisher";
 import { getDefinitelyTyped, AllPackages } from "@definitelytyped/definitions-parser";
 import {
@@ -17,13 +18,15 @@ import {
 import { readChangedPackages, ChangedPackages } from "./lib/versions";
 import { skipBadPublishes } from "./lib/npm";
 import { getSecret, Secret } from "./lib/secrets";
-import { cacheDirPath } from "./lib/settings";
 
 if (!module.parent) {
   const dry = !!yargs.argv.dry;
   const deprecateName = yargs.argv.deprecate as string | undefined;
   logUncaughtErrors(async () => {
-    const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]);
+    const dt = await getDefinitelyTyped(
+      process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions,
+      loggerWithErrors()[0]
+    );
     if (deprecateName !== undefined) {
       // A '--deprecate' command is available in case types-publisher got stuck *while* trying to deprecate a package.
       // Normally this should not be needed.
@@ -59,7 +62,7 @@ export default async function publishPackages(
     log("=== Publishing packages ===");
   }
 
-  const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined);
+  const client = await NpmPublishClient.create(dry ? "" : await getSecret(Secret.NPM_TOKEN), undefined);
 
   for (const cp of changedPackages.changedTypings) {
     log(`Publishing ${cp.pkg.desc}...`);
@@ -143,16 +146,12 @@ export default async function publishPackages(
     }
   }
 
-  await withNpmCache(
-    new UncachedNpmInfoClient(),
-    async (infoClient) => {
-      for (const n of changedPackages.changedNotNeededPackages) {
-        const target = skipBadPublishes(n, infoClient, log);
-        await publishNotNeededPackage(client, target, dry, log);
-      }
-    },
-    cacheDirPath
-  );
+  await withNpmCache(new UncachedNpmInfoClient(), async (infoClient) => {
+    for (const n of changedPackages.changedNotNeededPackages) {
+      const target = skipBadPublishes(n, infoClient, log);
+      await publishNotNeededPackage(client, target, dry, log);
+    }
+  });
 
   await writeLog("publishing.md", logResult());
   console.log("Done!");
diff --git a/packages/publisher/src/publish-registry.ts b/packages/publisher/src/publish-registry.ts
diff --git a/packages/retag/package.json b/packages/retag/package.json
diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts
diff --git a/packages/utils/src/npm.ts b/packages/utils/src/npm.ts
