Skip to content

Commit 2dfc38f

Browse files
fengmk2fengmk2
committed
feat: add XZ compression and decompression support
- Add XZ compression module with FileStream and UncompressStream classes - Implement compressFile() and uncompress() functions - Add TypeScript definitions for XZ module - Add comprehensive unit tests for all XZ functionality - Support compression levels 1-9 and multi-threading options - Fixes #96
1 parent b2d231b commit 2dfc38f

File tree

9 files changed

+538
-0
lines changed

9 files changed

+538
-0
lines changed

‎index.d.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,37 @@ export namespace gzip {
5353

5454
}
5555

56+
export namespace xz {
57+
58+
function compressFile(source: sourceType, dest: destType, opts?: any): Promise<void>
59+
60+
function uncompress(source: sourceType, dest: destType, opts?: any): Promise<void>
61+
62+
function decompress(source: sourceType, dest: destType, opts?: any): Promise<void>
63+
64+
export class FileStream extends ReadStream {
65+
66+
constructor(opts?: {
67+
preset?: number,
68+
threads?: number,
69+
source?: sourceType
70+
});
71+
72+
}
73+
74+
export class UncompressStream extends WriteStream {
75+
76+
constructor(opts?: {
77+
source?: sourceType
78+
});
79+
80+
on(event: string, listener: (...args: any[]) => void): this
81+
on(event: 'error', listener: (err: Error) => void): this
82+
83+
}
84+
85+
}
86+
5687
export namespace tar {
5788

5889
function compressFile(source: sourceType, dest: destType, opts?: any): Promise<void>

‎index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ exports.zip = require('./lib/zip');
44
exports.gzip = require('./lib/gzip');
55
exports.tar = require('./lib/tar');
66
exports.tgz = require('./lib/tgz');
7+
exports.xz = require('./lib/xz');

‎lib/xz/file_stream.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
'use strict';
2+
3+
const fs = require('fs');
4+
const lzma = require('lzma-native');
5+
const utils = require('../utils');
6+
const streamifier = require('streamifier');
7+
8+
class XzFileStream extends lzma.Compressor {
9+
constructor(opts) {
10+
opts = opts || {};
11+
const lzmaOpts = {
12+
preset: opts.preset || 6,
13+
threads: opts.threads || 0
14+
};
15+
super(lzmaOpts);
16+
17+
const sourceType = utils.sourceType(opts.source);
18+
19+
if (sourceType === 'file') {
20+
const stream = fs.createReadStream(opts.source, opts.fs);
21+
stream.on('error', err => this.emit('error', err));
22+
stream.pipe(this);
23+
return;
24+
}
25+
26+
if (sourceType === 'buffer') {
27+
const stream = streamifier.createReadStream(opts.source, opts.streamifier);
28+
stream.on('error', err => this.emit('error', err));
29+
stream.pipe(this);
30+
return;
31+
}
32+
33+
if (sourceType === 'stream') {
34+
opts.source.on('error', err => this.emit('error', err));
35+
opts.source.pipe(this);
36+
}
37+
38+
// else undefined: do nothing
39+
}
40+
}
41+
42+
module.exports = XzFileStream;

‎lib/xz/index.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
'use strict';
2+
3+
const utils = require('../utils');
4+
const XzFileStream = require('./file_stream');
5+
const XzUncompressStream = require('./uncompress_stream');
6+
7+
exports.FileStream = XzFileStream;
8+
exports.UncompressStream = XzUncompressStream;
9+
exports.compressFile = utils.makeFileProcessFn(XzFileStream);
10+
exports.uncompress = utils.makeFileProcessFn(XzUncompressStream);
11+
exports.decompress = utils.makeFileProcessFn(XzUncompressStream);

‎lib/xz/uncompress_stream.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
'use strict';
2+
3+
const fs = require('fs');
4+
const lzma = require('lzma-native');
5+
const utils = require('../utils');
6+
const streamifier = require('streamifier');
7+
8+
class XzUncompressStream extends lzma.Decompressor {
9+
constructor(opts) {
10+
opts = opts || {};
11+
super();
12+
13+
const sourceType = utils.sourceType(opts.source);
14+
15+
if (sourceType === 'file') {
16+
const stream = fs.createReadStream(opts.source, opts.fs);
17+
stream.on('error', err => this.emit('error', err));
18+
stream.pipe(this);
19+
return;
20+
}
21+
22+
if (sourceType === 'buffer') {
23+
const stream = streamifier.createReadStream(opts.source, opts.streamifier);
24+
stream.on('error', err => this.emit('error', err));
25+
stream.pipe(this);
26+
return;
27+
}
28+
29+
if (sourceType === 'stream') {
30+
opts.source.on('error', err => this.emit('error', err));
31+
opts.source.pipe(this);
32+
}
33+
34+
// else: waiting to be piped
35+
}
36+
}
37+
38+
module.exports = XzUncompressStream;

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
"flushwritable": "^1.0.0",
4444
"get-ready": "^1.0.0",
4545
"iconv-lite": "^0.5.0",
46+
"lzma-native": "^8.0.6",
4647
"streamifier": "^0.1.1",
4748
"tar-stream": "^1.5.2",
4849
"yazl": "^2.4.2"

‎test/xz/file_stream.test.js‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
const fs = require('fs');
2+
const os = require('os');
3+
const path = require('path');
4+
const uuid = require('uuid');
5+
const { pipeline: pump } = require('stream');
6+
const compressing = require('../..');
7+
const assert = require('assert');
8+
9+
describe('test/xz/file_stream.test.js', () => {
10+
it('should be a transform stream', done => {
11+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
12+
const sourceStream = fs.createReadStream(sourceFile);
13+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
14+
// console.log('destFile', destFile);
15+
const xzStream = new compressing.xz.FileStream();
16+
const destStream = fs.createWriteStream(destFile);
17+
pump(sourceStream, xzStream, destStream, err => {
18+
assert(!err);
19+
assert(fs.existsSync(destFile));
20+
done();
21+
});
22+
});
23+
24+
it('should compress according to file path', done => {
25+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
26+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
27+
// console.log('destFile', destFile);
28+
const xzStream = new compressing.xz.FileStream({ source: sourceFile });
29+
const destStream = fs.createWriteStream(destFile);
30+
pump(xzStream, destStream, err => {
31+
assert(!err);
32+
assert(fs.existsSync(destFile));
33+
done();
34+
});
35+
});
36+
37+
it('should compress file into Buffer', async () => {
38+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
39+
const xzStream = new compressing.xz.FileStream({ source: sourceFile });
40+
const xzChunks = [];
41+
for await (const chunk of xzStream) {
42+
xzChunks.push(chunk);
43+
}
44+
45+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
46+
await fs.promises.writeFile(destFile, Buffer.concat(xzChunks));
47+
// console.log(destFile);
48+
});
49+
50+
it('should compress buffer', done => {
51+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
52+
const sourceBuffer = fs.readFileSync(sourceFile);
53+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
54+
// console.log('destFile', destFile);
55+
const destStream = fs.createWriteStream(destFile);
56+
const xzStream = new compressing.xz.FileStream({ source: sourceBuffer });
57+
pump(xzStream, destStream, err => {
58+
assert(!err);
59+
assert(fs.existsSync(destFile));
60+
done();
61+
});
62+
63+
});
64+
65+
it('should compress stream', done => {
66+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
67+
const sourceStream = fs.createReadStream(sourceFile);
68+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
69+
// console.log('destFile', destFile);
70+
const destStream = fs.createWriteStream(destFile);
71+
const xzStream = new compressing.xz.FileStream({ source: sourceStream });
72+
pump(xzStream, destStream, err => {
73+
assert(!err);
74+
assert(fs.existsSync(destFile));
75+
done();
76+
});
77+
});
78+
79+
it('should compress with custom level', done => {
80+
const sourceFile = path.join(__dirname, '..', 'fixtures', 'xx.log');
81+
const destFile = path.join(os.tmpdir(), uuid.v4() + '.log.xz');
82+
const xzStream = new compressing.xz.FileStream({
83+
source: sourceFile,
84+
level: 6
85+
});
86+
const destStream = fs.createWriteStream(destFile);
87+
pump(xzStream, destStream, err => {
88+
assert(!err);
89+
assert(fs.existsSync(destFile));
90+
done();
91+
});
92+
});
93+
94+
it('should emit error if sourceFile does not exit', done => {
95+
const sourceFile = 'file-not-exist';
96+
const xzStream = new compressing.xz.FileStream({ source: sourceFile });
97+
xzStream.on('error', err => {
98+
assert(err);
99+
done();
100+
});
101+
});
102+
103+
it('should emit error if sourceStream emit error', done => {
104+
const sourceFile = 'file-not-exist';
105+
const sourceStream = fs.createReadStream(sourceFile);
106+
const xzStream = new compressing.xz.FileStream({ source: sourceStream });
107+
xzStream.on('error', err => {
108+
assert(err && err.code === 'ENOENT');
109+
done();
110+
});
111+
});
112+
113+
});

0 commit comments

Comments
 (0)