Move skipBadPublishes() -> calculate-versions

Author: jablko

packages/publisher/src/calculate-versions.ts

@@ -11,6 +11,7 @@ import {
 } from "@definitelytyped/utils";
 import { fetchTypesPackageVersionInfo } from "@definitelytyped/retag";
 import * as pacote from "pacote";
+import * as semver from "semver";
 
 if (!module.parent) {
   const log = loggerWithErrors()[0];
@@ -31,9 +32,9 @@ async function computeAndSaveChangedPackages(
   const cp = await computeChangedPackages(allPackages, log);
   const json: ChangedPackagesJson = {
     changedTypings: cp.changedTypings.map(
-      ({ pkg: { id }, version, latestVersion }): ChangedTypingJson => ({ id, version, latestVersion })
+      ({ pkg: { name }, version, latestVersion }): ChangedTypingJson => ({ name, version, latestVersion })
     ),
-    changedNotNeededPackages: cp.changedNotNeededPackages.map((p) => p.name),
+    changedNotNeededPackages: cp.changedNotNeededPackages.map(({ pkg: { name }, version }) => ({ name, version })),
   };
   await writeDataFile(versionsFilename, json);
   return cp;
@@ -66,7 +67,8 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE
   });
   log.info("# Computing deprecated packages...");
   const changedNotNeededPackages = await mapDefinedAsync(allPackages.allNotNeeded(), async (pkg) => {
-    if (!(await isAlreadyDeprecated(pkg, log))) {
+    const incipientVersion = await fetchIncipientStubVersion(pkg, log);
+    if (incipientVersion) {
       // Assert that dependencies (i.e. the replacement package) exist on npm.
       // Also checked in checkNotNeededPackage().
       await pacote.manifest(pkg.libraryName, { cache: defaultCacheDir }).catch((reason) => {
@@ -75,20 +77,39 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE
           : reason;
       });
       log.info(`To be deprecated: ${pkg.name}`);
-      return pkg;
+      return { pkg, version: incipientVersion };
     }
     return undefined;
   });
   return { changedTypings, changedNotNeededPackages };
 }
 
-async function isAlreadyDeprecated(pkg: NotNeededPackage, log: LoggerWithErrors): Promise<unknown> {
-  const offline = await pacote.manifest(pkg.fullNpmName, { cache: defaultCacheDir, offline: true }).catch((reason) => {
-    if (reason.code !== "ENOTCACHED") throw reason;
-    return undefined;
-  });
-  if (offline?.deprecated) return offline.deprecated;
-  log.info(`Version info not cached for deprecated package ${pkg.desc}`);
-  const online = await pacote.manifest(pkg.fullNpmName, { cache: defaultCacheDir, preferOnline: true });
-  return online.deprecated;
+/**
+ * Return the version of the stub @types package we're about to publish and deprecate, if we haven't already deprecated that package.
+ * It's the max of the not-needed version and the max published version + 1, in case we already published a not-needed-versioned stub but failed to deprecate it, for whatever reason.
+ */
+export async function fetchIncipientStubVersion(pkg: NotNeededPackage, log: LoggerWithErrors): Promise<string | false> {
+  const packument = await revalidate();
+  return (
+    !packument.versions[packument["dist-tags"].latest].deprecated &&
+    String(
+      semver.maxSatisfying(
+        [pkg.version, semver.inc(semver.maxSatisfying(Object.keys(packument.versions), "*")!, "patch")!],
+        "*"
+      )
+    )
+  );
+
+  async function revalidate() {
+    const offline = await pacote
+      .packument(pkg.fullNpmName, { cache: defaultCacheDir, offline: true })
+      .catch((reason) => {
+        if (reason.code !== "ENOTCACHED") throw reason;
+        return undefined;
+      });
+    if (offline?.versions[offline["dist-tags"].latest].deprecated) return offline;
+    log.info(`Version info not cached for deprecated package ${pkg.desc}`);
+    const online = await pacote.packument(pkg.fullNpmName, { cache: defaultCacheDir, preferOnline: true });
+    return online;
+  }
 }

packages/publisher/src/generate-packages.ts

@@ -15,7 +15,6 @@ import {
   logger,
   writeLog,
   writeFile,
-  Logger,
   defaultCacheDir,
   writeTgz,
 } from "@definitelytyped/utils";
@@ -34,7 +33,6 @@ import {
 import * as pacote from "pacote";
 import { readChangedPackages, ChangedPackages } from "./lib/versions";
 import { outputDirectory } from "./util/util";
-import { skipBadPublishes } from "./lib/npm";
 
 const mitLicense = readFileSync(joinPaths(__dirname, "..", "LICENSE"), "utf-8");
 
@@ -68,9 +66,9 @@ export default async function generatePackages(
     log(` * ${pkg.desc}`);
   }
   log("## Generating deprecated packages");
-  for (const pkg of changedPackages.changedNotNeededPackages) {
+  for (const { pkg, version } of changedPackages.changedNotNeededPackages) {
     log(` * ${pkg.libraryName}`);
-    await generateNotNeededPackage(pkg, log);
+    await generateNotNeededPackage(pkg, version);
   }
   await writeLog("package-generator.md", logResult());
 }
@@ -92,12 +90,11 @@ async function generateTypingPackage(
   );
 }
 
-async function generateNotNeededPackage(pkg: NotNeededPackage, log: Logger): Promise<void> {
-  pkg = await skipBadPublishes(pkg, log);
+async function generateNotNeededPackage(pkg: NotNeededPackage, version: string): Promise<void> {
   const info = await pacote.manifest(pkg.libraryName, { cache: defaultCacheDir, fullMetadata: true });
   const readme = `This is a stub types definition for ${getFullNpmName(pkg.name)} (${info.homepage}).\n
 ${pkg.libraryName} provides its own type definitions, so you don't need ${getFullNpmName(pkg.name)} installed!`;
-  await writeCommonOutputs(pkg, createNotNeededPackageJSON(pkg), readme);
+  await writeCommonOutputs(pkg, createNotNeededPackageJSON(pkg, version), readme);
 }
 
 async function writeCommonOutputs(pkg: AnyPackage, packageJson: string, readme: string): Promise<void> {
@@ -197,10 +194,13 @@ function dependencySemver(dependency: DependencyVersion): string {
   return dependency === "*" ? dependency : "^" + formatTypingVersion(dependency);
 }
 
-export function createNotNeededPackageJSON({ libraryName, license, fullNpmName, version }: NotNeededPackage): string {
+export function createNotNeededPackageJSON(
+  { libraryName, license, fullNpmName }: NotNeededPackage,
+  version: string
+): string {
   const out = {
     name: fullNpmName,
-    version: String(version),
+    version,
     typings: null, // tslint:disable-line no-null-keyword
     description: `Stub TypeScript definitions entry for ${libraryName}, which provides its own types definitions`,
     main: "",

packages/publisher/src/lib/npm.ts

@@ -1,13 +1,13 @@
 import assert = require("assert");
 import { Logger, joinPaths, readFileAndWarn, NpmPublishClient } from "@definitelytyped/utils";
-import { NotNeededPackage, AnyPackage } from "@definitelytyped/definitions-parser";
+import { NotNeededPackage, AnyPackage, TypingsData } from "@definitelytyped/definitions-parser";
 import { updateTypeScriptVersionTags, updateLatestTag } from "@definitelytyped/retag";
 import { ChangedTyping } from "./versions";
 import { outputDirectory } from "../util/util";
 
 export async function publishTypingsPackage(
   client: NpmPublishClient,
-  changedTyping: ChangedTyping,
+  changedTyping: ChangedTyping<TypingsData>,
   dry: boolean,
   log: Logger
 ): Promise<void> {
@@ -28,13 +28,14 @@ export async function publishTypingsPackage(
 export async function publishNotNeededPackage(
   client: NpmPublishClient,
   pkg: NotNeededPackage,
+  version: string,
   dry: boolean,
   log: Logger
 ): Promise<void> {
   log(`Deprecating ${pkg.name}`);
   await common(client, pkg, log, dry);
   // Don't use a newline in the deprecation message because it will be displayed as "\n" and not as a newline.
-  await deprecateNotNeededPackage(client, pkg, dry, log);
+  await deprecateNotNeededPackage(client, pkg, version, dry, log);
 }
 
 async function common(client: NpmPublishClient, pkg: AnyPackage, log: Logger, dry: boolean): Promise<void> {
@@ -46,14 +47,15 @@ async function common(client: NpmPublishClient, pkg: AnyPackage, log: Logger, dr
 export async function deprecateNotNeededPackage(
   client: NpmPublishClient,
   pkg: NotNeededPackage,
+  version: string,
   dry = false,
   log: Logger
 ): Promise<void> {
   const name = pkg.fullNpmName;
   if (dry) {
-    log("(dry) Skip deprecate not needed package " + name + " at " + pkg.version);
+    log("(dry) Skip deprecate not needed package " + name + " at " + version);
   } else {
-    log(`Deprecating ${name} at ${pkg.version} with message: ${pkg.deprecatedMessage()}.`);
-    await client.deprecate(name, String(pkg.version), pkg.deprecatedMessage());
+    log(`Deprecating ${name} at ${version} with message: ${pkg.deprecatedMessage()}.`);
+    await client.deprecate(name, version, pkg.deprecatedMessage());
   }
 }

packages/publisher/src/lib/package-publisher.ts

@@ -1,50 +1,46 @@
 import { assertDefined } from "@definitelytyped/utils";
-import {
-  AllPackages,
-  NotNeededPackage,
-  PackageId,
-  TypingsData,
-  readDataFile,
-} from "@definitelytyped/definitions-parser";
+import { AllPackages, NotNeededPackage, TypingsData, readDataFile } from "@definitelytyped/definitions-parser";
+import * as semver from "semver";
 
 export const versionsFilename = "versions.json";
 
-export interface ChangedTyping {
-  readonly pkg: TypingsData;
+export interface ChangedTyping<T> {
+  readonly pkg: T;
   /** This is the version to be published, meaning it's the version that doesn't exist yet. */
   readonly version: string;
   /** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
-  readonly latestVersion: string | undefined;
+  readonly latestVersion?: string;
 }
 
 export interface ChangedPackagesJson {
   readonly changedTypings: readonly ChangedTypingJson[];
-  readonly changedNotNeededPackages: readonly string[];
+  readonly changedNotNeededPackages: readonly ChangedTypingJson[];
 }
 
 export interface ChangedTypingJson {
-  readonly id: PackageId;
+  readonly name: string;
   readonly version: string;
   readonly latestVersion?: string;
 }
 
 export interface ChangedPackages {
-  readonly changedTypings: readonly ChangedTyping[];
-  readonly changedNotNeededPackages: readonly NotNeededPackage[];
+  readonly changedTypings: readonly ChangedTyping<TypingsData>[];
+  readonly changedNotNeededPackages: readonly ChangedTyping<NotNeededPackage>[];
 }
 
 export async function readChangedPackages(allPackages: AllPackages): Promise<ChangedPackages> {
   const json = (await readDataFile("calculate-versions", versionsFilename)) as ChangedPackagesJson;
   return {
     changedTypings: json.changedTypings.map(
-      ({ id, version, latestVersion }): ChangedTyping => ({
-        pkg: allPackages.getTypingsData(id),
+      ({ name, version, latestVersion }): ChangedTyping<TypingsData> => ({
+        pkg: allPackages.getTypingsData({ name, version: new semver.SemVer(version) }),
         version,
         latestVersion,
       })
     ),
-    changedNotNeededPackages: json.changedNotNeededPackages.map((id) =>
-      assertDefined(allPackages.getNotNeededPackage(id))
-    ),
+    changedNotNeededPackages: json.changedNotNeededPackages.map(({ name, version }) => ({
+      pkg: assertDefined(allPackages.getNotNeededPackage(name)),
+      version,
+    })),
   };
 }

packages/publisher/src/lib/versions.ts

@@ -13,7 +13,6 @@ import {
   NpmPublishClient,
 } from "@definitelytyped/utils";
 import { readChangedPackages, ChangedPackages } from "./lib/versions";
-import { skipBadPublishes } from "./lib/npm";
 import { getSecret, Secret } from "./lib/secrets";
 
 if (!module.parent) {
@@ -24,11 +23,12 @@ if (!module.parent) {
     if (deprecateName !== undefined) {
       // A '--deprecate' command is available in case types-publisher got stuck *while* trying to deprecate a package.
       // Normally this should not be needed.
-
+      const pkg = AllPackages.readSingleNotNeeded(deprecateName, dt);
       const log = logger()[0];
       await deprecateNotNeededPackage(
         await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined),
-        AllPackages.readSingleNotNeeded(deprecateName, dt),
+        pkg,
+        String(pkg.version),
         /*dry*/ false,
         log
       );
@@ -140,9 +140,8 @@ export default async function publishPackages(
     }
   }
 
-  for (const n of changedPackages.changedNotNeededPackages) {
-    const target = await skipBadPublishes(n, log);
-    await publishNotNeededPackage(client, target, dry, log);
+  for (const { pkg, version } of changedPackages.changedNotNeededPackages) {
+    await publishNotNeededPackage(client, pkg, version, dry, log);
   }
 
   await writeLog("publishing.md", logResult());

packages/publisher/src/publish-packages.ts

@@ -0,0 +1,44 @@
+import * as util from "util";
+import { NotNeededPackage } from "@definitelytyped/definitions-parser";
+import { fetchIncipientStubVersion } from "../src/calculate-versions";
+
+jest.mock("pacote", () => ({
+  async packument(spec: string) {
+    switch (spec) {
+      case "@types/not-needed": // A not-needed @types package about to be deprecated.
+        return {
+          "dist-tags": { latest: "1.0.1" },
+          versions: { "1.0.0": {}, "1.0.1": {} },
+        };
+      case "@types/bad-publish": // A not-needed package after a bad publish.
+        return {
+          "dist-tags": { latest: "1.2.3" },
+          versions: { "1.0.0": {}, "1.0.1": {}, "1.2.3": {} },
+        };
+      case "@types/already-deprecated": // A successfully deprecated not-needed stub package.
+        return {
+          "dist-tags": { latest: "1.2.3" },
+          versions: { "1.0.0": {}, "1.0.1": {}, "1.2.3": { deprecated: "Contains built-in declarations." } },
+        };
+    }
+    throw new Error(`Unexpected npm registry fetch: ${util.inspect(spec)}`);
+  },
+}));
+
+const notNeeded = new NotNeededPackage("not-needed", "not-needed", "1.2.3");
+const badPublish = new NotNeededPackage("bad-publish", "bad-publish", "1.2.3");
+const alreadyDeprecated = new NotNeededPackage("already-deprecated", "already-deprecated", "1.2.3");
+
+const log = { info() {}, error() {} };
+
+test("Returns the not-needed version", () => {
+  return expect(fetchIncipientStubVersion(notNeeded, log)).resolves.toBe("1.2.3");
+});
+
+test("Skips bad publishes", () => {
+  return expect(fetchIncipientStubVersion(badPublish, log)).resolves.toBe("1.2.4");
+});
+
+test("Ignores already-deprecated packages", () => {
+  return expect(fetchIncipientStubVersion(alreadyDeprecated, log)).resolves.toBe(false);
+});

packages/publisher/test/calculate-versions.test.ts

@@ -137,7 +137,7 @@ testo({
 }`);
   },
   basicNotNeededPackageJson() {
-    const s = createNotNeededPackageJSON(createUnneededPackage());
+    const s = createNotNeededPackageJSON(createUnneededPackage(), "1.1.1");
     expect(s).toEqual(`{
     "name": "@types/absalom",
     "version": "1.1.1",
@@ -154,7 +154,7 @@ testo({
   },
   scopedNotNeededPackageJson() {
     const scopedUnneeded = new NotNeededPackage("google-cloud__pubsub", "@google-cloud/chubdub", "0.26.0");
-    const s = createNotNeededPackageJSON(scopedUnneeded);
+    const s = createNotNeededPackageJSON(scopedUnneeded, "0.26.0");
     expect(s).toEqual(`{
     "name": "@types/google-cloud__pubsub",
     "version": "0.26.0",