From 01e2d19613af82dfbd6946bccda58b4a7693f786 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Tue, 14 Oct 2025 20:45:13 +0800 Subject: [PATCH 1/2] feat: sls use lz4-napi --- alibabacloud-gateway-sls/ts/package.json | 4 ++-- alibabacloud-gateway-sls/util/ts/package.json | 5 ++--- alibabacloud-gateway-sls/util/ts/src/compress.ts | 6 ++---- alibabacloud-gateway-sls/util/ts/src/decompress.ts | 11 +++++------ 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/alibabacloud-gateway-sls/ts/package.json b/alibabacloud-gateway-sls/ts/package.json index c23f19a3..f86991c2 100644 --- a/alibabacloud-gateway-sls/ts/package.json +++ b/alibabacloud-gateway-sls/ts/package.json @@ -1,6 +1,6 @@ { "name": "@alicloud/gateway-sls", - "version": "0.3.0", + "version": "0.3.1", "description": "", "main": "dist/client.js", "scripts": { @@ -29,7 +29,7 @@ "@alicloud/darabonba-array": "^0.1.0", "@alicloud/darabonba-encode-util": "^0.0.2", "@alicloud/darabonba-signature-util": "^0.0.4", - "@alicloud/gateway-sls-util": "^0.3.0" + "@alicloud/gateway-sls-util": "^0.3.1" }, "files": [ "dist", diff --git a/alibabacloud-gateway-sls/util/ts/package.json b/alibabacloud-gateway-sls/util/ts/package.json index 70560501..871862f4 100644 --- a/alibabacloud-gateway-sls/util/ts/package.json +++ b/alibabacloud-gateway-sls/util/ts/package.json @@ -1,6 +1,6 @@ { "name": "@alicloud/gateway-sls-util", - "version": "0.3.0", + "version": "0.3.1", "description": "", "main": "dist/client.js", "scripts": { @@ -12,7 +12,6 @@ "author": "", "license": "ISC", "devDependencies": { - "@types/lz4": "^0.6.4", "@types/node": "^12.12.26", "nyc": "^15.0.0", "source-map-support": "^0.5.16", @@ -20,7 +19,7 @@ "typescript": "^3.7.5" }, "dependencies": { - "lz4": "^0.6.5" + "lz4-napi": "^2.9.0" }, "files": [ "dist", diff --git a/alibabacloud-gateway-sls/util/ts/src/compress.ts b/alibabacloud-gateway-sls/util/ts/src/compress.ts index c7530acc..c5e9bff7 100644 --- a/alibabacloud-gateway-sls/util/ts/src/compress.ts +++ b/alibabacloud-gateway-sls/util/ts/src/compress.ts @@ -1,12 +1,10 @@ -import lz4 from 'lz4'; +import {compress as napiLz4Compress} from 'lz4-napi'; import zlib from 'zlib'; const supportedCompressor = ['lz4', 'gzip', 'deflate'] export async function lz4Compress(data: Buffer): Promise { - const output = Buffer.alloc(lz4.encodeBound(data.length)); - const n = lz4.encodeBlock(data, output); - return Buffer.from(output.slice(0, n)); + return await napiLz4Compress(data); } export async function gzipCompress(data: Buffer): Promise { diff --git a/alibabacloud-gateway-sls/util/ts/src/decompress.ts b/alibabacloud-gateway-sls/util/ts/src/decompress.ts index 4c3157d2..3641b1e8 100644 --- a/alibabacloud-gateway-sls/util/ts/src/decompress.ts +++ b/alibabacloud-gateway-sls/util/ts/src/decompress.ts @@ -1,6 +1,6 @@ import { Readable } from 'stream'; import { readableStreamToBuffer } from './common'; -import lz4 from 'lz4'; +import {uncompress as napiLz4Uncompress} from 'lz4-napi'; import zlib from 'zlib'; const supportedDecompressor = ['lz4', 'gzip', 'deflate'] @@ -10,12 +10,11 @@ export async function isDecompressorAvailable(compressType: string): Promise { - const output = Buffer.alloc(bodyRawSize); - const n = lz4.decodeBlock(await readableStreamToBuffer(input), output); - if (n !== bodyRawSize) { - return Buffer.from(output.slice(0, n)); + const napiOutput = await napiLz4Uncompress(await readableStreamToBuffer(input)); + if (napiOutput.length !== bodyRawSize) { + return Buffer.from(napiOutput.slice(0, bodyRawSize)); } - return output; + return napiOutput; } export async function gzipDecompress(input: Readable, bodyRawSize: number): Promise { From 2c177933d381ba48a280f969355d2f961eb6c6e5 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Wed, 15 Oct 2025 10:36:37 +0800 Subject: [PATCH 2/2] fix prepend size --- .../util/ts/src/common.ts | 19 ++++++++++++------- .../util/ts/src/compress.ts | 3 ++- .../util/ts/src/decompress.ts | 12 ++++++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/alibabacloud-gateway-sls/util/ts/src/common.ts b/alibabacloud-gateway-sls/util/ts/src/common.ts index ea7bf15b..0bb67a59 100644 --- a/alibabacloud-gateway-sls/util/ts/src/common.ts +++ b/alibabacloud-gateway-sls/util/ts/src/common.ts @@ -2,11 +2,16 @@ import { Readable } from 'stream'; export async function readableStreamToBuffer(stream: Readable): Promise { const chunks: Buffer[] = []; - return new Promise((resolve, reject) => { - stream.on('data', (chunk: Buffer) => chunks.push(chunk)); - stream.on('error', reject); - stream.on('end', () => { - resolve(Buffer.concat(chunks)); - }); - }); + for await (const chunk of stream) { + chunks.push(chunk as Buffer); + } + return Buffer.concat(chunks); +} + +export async function readableStreamWithPrependBuffer(stream: Readable, prependBuffer: Buffer): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(chunk as Buffer); + } + return Buffer.concat([prependBuffer, ...chunks]); } \ No newline at end of file diff --git a/alibabacloud-gateway-sls/util/ts/src/compress.ts b/alibabacloud-gateway-sls/util/ts/src/compress.ts index c5e9bff7..8f0d6728 100644 --- a/alibabacloud-gateway-sls/util/ts/src/compress.ts +++ b/alibabacloud-gateway-sls/util/ts/src/compress.ts @@ -4,7 +4,8 @@ import zlib from 'zlib'; const supportedCompressor = ['lz4', 'gzip', 'deflate'] export async function lz4Compress(data: Buffer): Promise { - return await napiLz4Compress(data); + // eliminate 4 bytes prepended size + return (await napiLz4Compress(data)).slice(4); } export async function gzipCompress(data: Buffer): Promise { diff --git a/alibabacloud-gateway-sls/util/ts/src/decompress.ts b/alibabacloud-gateway-sls/util/ts/src/decompress.ts index 3641b1e8..16b1823c 100644 --- a/alibabacloud-gateway-sls/util/ts/src/decompress.ts +++ b/alibabacloud-gateway-sls/util/ts/src/decompress.ts @@ -1,8 +1,8 @@ import { Readable } from 'stream'; -import { readableStreamToBuffer } from './common'; +import { readableStreamToBuffer, readableStreamWithPrependBuffer } from './common'; import {uncompress as napiLz4Uncompress} from 'lz4-napi'; import zlib from 'zlib'; - +import fs from 'fs'; const supportedDecompressor = ['lz4', 'gzip', 'deflate'] export async function isDecompressorAvailable(compressType: string): Promise { @@ -10,9 +10,13 @@ export async function isDecompressorAvailable(compressType: string): Promise { - const napiOutput = await napiLz4Uncompress(await readableStreamToBuffer(input)); + // prepend 4 bytes size to the buffer + const sizeBuffer = Buffer.alloc(4); + sizeBuffer.writeUInt32LE(bodyRawSize, 0); + let compressedBuffer = await readableStreamWithPrependBuffer(input, sizeBuffer); + const napiOutput = await napiLz4Uncompress(compressedBuffer); if (napiOutput.length !== bodyRawSize) { - return Buffer.from(napiOutput.slice(0, bodyRawSize)); + throw new Error(`unexpected uncompressed size: ${napiOutput.length}, expected: ${bodyRawSize}`); } return napiOutput; }