fix(lib-storage): use input parameter object size information if available (#7385)

* fix(lib-storage): use input parameter object size information if available

# Conflicts:
#	lib/lib-storage/src/Upload.ts

* fix(lib-storage): use only ContentLength

* fix: import ordering

* fix: byteLengthSource

* fix: add error message

Author: kuhe

lib/lib-storage/src/Upload.spec.ts

@@ -1,11 +1,12 @@
+import { EventEmitter, Readable } from "stream";
 import { afterAll, afterEach, beforeEach, describe, expect, test as it, vi } from "vitest";
 
+import { Progress, Upload } from "./index";
+
 /* eslint-disable no-var */
 var hostname = "s3.region.amazonaws.com";
 var port: number | undefined;
 
-import { EventEmitter, Readable } from "stream";
-
 vi.mock("@aws-sdk/client-s3", async () => {
   const sendMock = vi.fn().mockImplementation(async (x) => x);
   const endpointMock = vi.fn().mockImplementation(() => ({
@@ -73,12 +74,20 @@ import {
 } from "@aws-sdk/client-s3";
 import { AbortController } from "@smithy/abort-controller";
 
-import { Progress, Upload } from "./index";
-
 const DEFAULT_PART_SIZE = 1024 * 1024 * 5;
 
+type Expose = {
+  totalBytes: number | undefined;
+};
+type VisibleForTesting = Omit<Upload, keyof Expose> & Expose;
+
 describe(Upload.name, () => {
-  const s3MockInstance = new S3Client();
+  const s3MockInstance = new S3Client({
+    credentials: {
+      accessKeyId: "UNIT",
+      secretAccessKey: "UNIT",
+    },
+  });
 
   beforeEach(() => {
     vi.clearAllMocks();
@@ -107,6 +116,29 @@ describe(Upload.name, () => {
     Body: "this-is-a-sample-payload",
   };
 
+  it("uses the input parameters for object length if provided", async () => {
+    let upload = new Upload({
+      params: {
+        Bucket: "",
+        Key: "",
+        Body: Buffer.from("a".repeat(256)),
+        ContentLength: 6 * 1024 * 1024,
+      },
+      client: s3MockInstance,
+    }) as unknown as VisibleForTesting;
+    expect(upload.totalBytes).toEqual(6 * 1024 * 1024);
+
+    upload = new Upload({
+      params: {
+        Bucket: "",
+        Key: "",
+        Body: Buffer.from("a".repeat(256)),
+      },
+      client: s3MockInstance,
+    }) as unknown as VisibleForTesting;
+    expect(upload.totalBytes).toEqual(256);
+  });
+
   it("correctly exposes the event emitter API", () => {
     const upload = new Upload({
       params,
@@ -937,7 +969,7 @@ describe(Upload.name, () => {
           partSize: 1024, // Too small
           client: new S3({}),
         });
-      }).toThrow(/EntityTooSmall: Your proposed upload partsize/);
+      }).toThrow(/EntityTooSmall: Your proposed upload part size/);
     });
 
     it("should validate minimum queueSize", () => {

lib/lib-storage/src/Upload.ts

@@ -3,15 +3,18 @@ import {
   ChecksumAlgorithm,
   CompletedPart,
   CompleteMultipartUploadCommand,
+  CompleteMultipartUploadCommandInput,
   CompleteMultipartUploadCommandOutput,
   CreateMultipartUploadCommand,
+  CreateMultipartUploadCommandInput,
   CreateMultipartUploadCommandOutput,
   PutObjectCommand,
   PutObjectCommandInput,
   PutObjectTaggingCommand,
   S3Client,
   Tag,
   UploadPartCommand,
+  UploadPartCommandInput,
 } from "@aws-sdk/client-s3";
 import { AbortController } from "@smithy/abort-controller";
 import {
@@ -24,7 +27,8 @@ import { extendedEncodeURIComponent } from "@smithy/smithy-client";
 import type { AbortController as IAbortController, AbortSignal as IAbortSignal, Endpoint } from "@smithy/types";
 import { EventEmitter } from "events";
 
-import { byteLength } from "./bytelength";
+import { byteLength } from "./byteLength";
+import { BYTE_LENGTH_SOURCE, byteLengthSource } from "./byteLengthSource";
 import { getChunk } from "./chunker";
 import { BodyDataTypes, Options, Progress } from "./types";
 
@@ -52,10 +56,12 @@ export class Upload extends EventEmitter {
   private readonly tags: Tag[] = [];
 
   private readonly client: S3Client;
-  private readonly params: PutObjectCommandInput;
+  private readonly params: PutObjectCommandInput &
+    Partial<CreateMultipartUploadCommandInput & UploadPartCommandInput & CompleteMultipartUploadCommandInput>;
 
   // used for reporting progress.
   private totalBytes?: number;
+  private readonly totalBytesSource?: BYTE_LENGTH_SOURCE;
   private bytesUploadedSoFar: number;
 
   // used in the upload.
@@ -93,13 +99,18 @@ export class Upload extends EventEmitter {
     }
 
     // set progress defaults
-    this.totalBytes = byteLength(this.params.Body);
+    this.totalBytes = this.params.ContentLength ?? byteLength(this.params.Body);
+    this.totalBytesSource = byteLengthSource(this.params.Body, this.params.ContentLength);
     this.bytesUploadedSoFar = 0;
     this.abortController = options.abortController ?? new AbortController();
 
     this.partSize =
       options.partSize || Math.max(Upload.MIN_PART_SIZE, Math.floor((this.totalBytes || 0) / this.MAX_PARTS));
-    this.expectedPartsCount = this.totalBytes !== undefined ? Math.ceil(this.totalBytes / this.partSize) : undefined;
+
+    if (this.totalBytes !== undefined) {
+      this.expectedPartsCount = Math.ceil(this.totalBytes / this.partSize);
+    }
+
     this.__validateInput();
   }
 
@@ -373,9 +384,14 @@ export class Upload extends EventEmitter {
 
     let result;
     if (this.isMultiPart) {
-      const { expectedPartsCount, uploadedParts } = this;
-      if (expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) {
-        throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s).`);
+      const { expectedPartsCount, uploadedParts, totalBytes, totalBytesSource } = this;
+      if (totalBytes !== undefined && expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) {
+        throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s).
+The expected part count is based on the byte-count of the input.params.Body,
+which was read from ${totalBytesSource} and is ${totalBytes}.
+If this is not correct, provide an override value by setting a number
+to input.params.ContentLength in bytes.
+`);
       }
 
       this.uploadedParts.sort((a, b) => a.PartNumber! - b.PartNumber!);
@@ -470,7 +486,7 @@ export class Upload extends EventEmitter {
 
     if (this.partSize < Upload.MIN_PART_SIZE) {
       throw new Error(
-        `EntityTooSmall: Your proposed upload partsize [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)`
+        `EntityTooSmall: Your proposed upload part size [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)`
       );
     }
 

lib/lib-storage/src/byteLength.spec.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { BYTE_LENGTH_SOURCE, byteLengthSource } from "./byteLengthSource";
+import { runtimeConfig } from "./runtimeConfig";
+
+vi.mock("./runtimeConfig", () => ({
+  runtimeConfig: {
+    lstatSync: vi.fn(),
+  },
+}));
+
+describe("byteLengthSource", () => {
+  it("should return CONTENT_LENGTH when override is provided", () => {
+    expect(byteLengthSource({}, 100)).toBe(BYTE_LENGTH_SOURCE.CONTENT_LENGTH);
+  });
+
+  it("should return EMPTY_INPUT for null input", () => {
+    expect(byteLengthSource(null)).toBe(BYTE_LENGTH_SOURCE.EMPTY_INPUT);
+  });
+
+  it("should return EMPTY_INPUT for undefined input", () => {
+    expect(byteLengthSource(undefined)).toBe(BYTE_LENGTH_SOURCE.EMPTY_INPUT);
+  });
+
+  it("should return STRING_LENGTH for string input", () => {
+    expect(byteLengthSource("test")).toBe(BYTE_LENGTH_SOURCE.STRING_LENGTH);
+  });
+
+  it("should return TYPED_ARRAY for input with byteLength", () => {
+    const input = new Uint8Array(10);
+    expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.TYPED_ARRAY);
+  });
+
+  it("should return LENGTH for input with length property", () => {
+    const input = { length: 10 };
+    expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.LENGTH);
+  });
+
+  it("should return SIZE for input with size property", () => {
+    const input = { size: 10 };
+    expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.SIZE);
+  });
+
+  it("should return START_END_DIFF for input with start and end properties", () => {
+    const input = { start: 0, end: 10 };
+    expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.START_END_DIFF);
+  });
+
+  it("should return LSTAT for input with path that exists", () => {
+    const input = { path: "/test/path" };
+    vi.mocked(runtimeConfig.lstatSync).mockReturnValue({ size: 100 } as any);
+
+    expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.LSTAT);
+    expect(runtimeConfig.lstatSync).toHaveBeenCalledWith("/test/path");
+  });
+
+  it("should return undefined for input with path that throws error", () => {
+    const input = { path: "/test/path" };
+    vi.mocked(runtimeConfig.lstatSync).mockImplementation(() => {
+      throw new Error("File not found");
+    });
+
+    expect(byteLengthSource(input)).toBeUndefined();
+  });
+
+  it("should return undefined for input with no matching properties", () => {
+    const input = { foo: "bar" };
+    expect(byteLengthSource(input)).toBeUndefined();
+  });
+});

lib/lib-storage/src/byteLength.ts

@@ -0,0 +1,41 @@
+import { Buffer } from "buffer"; // do not remove this import: Node.js buffer or buffer NPM module for browser.
+
+import { runtimeConfig } from "./runtimeConfig";
+
+/**
+ * Clients use util-body-length-[node|browser] instead.
+ * @internal
+ * @param input - to examine.
+ * @returns byte count of input or undefined if indeterminable.
+ */
+export const byteLength = (input: any): number | undefined => {
+  if (input == null) {
+    return 0;
+  }
+
+  if (typeof input === "string") {
+    return Buffer.byteLength(input);
+  }
+
+  if (typeof input.byteLength === "number") {
+    // Uint8Array, ArrayBuffer, Buffer, and ArrayBufferView
+    return input.byteLength;
+  } else if (typeof input.length === "number") {
+    // todo: unclear in what cases this is a valid byte count.
+    return input.length;
+  } else if (typeof input.size === "number") {
+    // todo: unclear in what cases this is a valid byte count.
+    return input.size;
+  } else if (typeof input.start === "number" && typeof input.end === "number") {
+    // file read stream with range.
+    return input.end + 1 - input.start;
+  } else if (typeof input.path === "string") {
+    // file read stream with path.
+    try {
+      return runtimeConfig.lstatSync(input.path).size;
+    } catch (error) {
+      return undefined;
+    }
+  }
+  return undefined;
+};

lib/lib-storage/src/byteLengthSource.ts

@@ -0,0 +1,54 @@
+import { runtimeConfig } from "./runtimeConfig";
+
+/**
+ * @internal
+ */
+export enum BYTE_LENGTH_SOURCE {
+  EMPTY_INPUT = "a null or undefined Body",
+  CONTENT_LENGTH = "the ContentLength property of the params set by the caller",
+  STRING_LENGTH = "the encoded byte length of the Body string",
+  TYPED_ARRAY = "the byteLength of a typed byte array such as Uint8Array",
+  LENGTH = "the value of Body.length",
+  SIZE = "the value of Body.size",
+  START_END_DIFF = "the numeric difference between Body.start and Body.end",
+  LSTAT = "the size of the file given by Body.path on disk as reported by lstatSync",
+}
+
+/**
+ * The returned value should complete the sentence, "The byte count of the data was determined by ...".
+ * @internal
+ * @param input - to examine.
+ * @param override - manually specified value.
+ * @returns source of byte count information.
+ */
+export const byteLengthSource = (input: any, override?: number): BYTE_LENGTH_SOURCE | undefined => {
+  if (override != null) {
+    return BYTE_LENGTH_SOURCE.CONTENT_LENGTH;
+  }
+
+  if (input == null) {
+    return BYTE_LENGTH_SOURCE.EMPTY_INPUT;
+  }
+
+  if (typeof input === "string") {
+    return BYTE_LENGTH_SOURCE.STRING_LENGTH;
+  }
+
+  if (typeof input.byteLength === "number") {
+    return BYTE_LENGTH_SOURCE.TYPED_ARRAY;
+  } else if (typeof input.length === "number") {
+    return BYTE_LENGTH_SOURCE.LENGTH;
+  } else if (typeof input.size === "number") {
+    return BYTE_LENGTH_SOURCE.SIZE;
+  } else if (typeof input.start === "number" && typeof input.end === "number") {
+    return BYTE_LENGTH_SOURCE.START_END_DIFF;
+  } else if (typeof input.path === "string") {
+    try {
+      runtimeConfig.lstatSync(input.path).size;
+      return BYTE_LENGTH_SOURCE.LSTAT;
+    } catch (error) {
+      return undefined;
+    }
+  }
+  return undefined;
+};