Add support for zstd

PR-URL: #35
Credit: @JamieMagee
Close: #35
Reviewed-by: @isaacs

Author: JamieMagee

README.md

@@ -41,6 +41,8 @@ provided by that class.
 - Unzip
 - BrotliCompress (Node v10 and higher)
 - BrotliDecompress (Node v10 and higher)
+- ZstdCompress (Node v22.15 and higher)
+- ZstdDecompress (Node v22.15 and higher)
 
 ## USAGE
 

package-lock.json

@@ -33,7 +33,7 @@
   "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)",
   "license": "MIT",
   "devDependencies": {
-    "@types/node": "^22.13.14",
+    "@types/node": "^22.15.29",
     "tap": "^21.1.0",
     "tshy": "^3.0.2",
     "typedoc": "^0.28.1"

package.json

@@ -67,6 +67,7 @@ export type ZlibHandle =
   | realZlib.DeflateRaw
   | realZlib.InflateRaw
 export type BrotliMode = 'BrotliCompress' | 'BrotliDecompress'
+export type ZstdMode = 'ZstdCompress' | 'ZstdDecompress'
 
 abstract class ZlibBase extends Minipass<Buffer, ChunkWithFlushFlag> {
   #sawError: boolean = false
@@ -89,7 +90,7 @@ abstract class ZlibBase extends Minipass<Buffer, ChunkWithFlushFlag> {
   }
   /* c8 ignore stop */
 
-  constructor(opts: ZlibBaseOptions, mode: ZlibMode | BrotliMode) {
+  constructor(opts: ZlibBaseOptions, mode: ZlibMode | BrotliMode | ZstdMode) {
     if (!opts || typeof opts !== 'object')
       throw new TypeError('invalid options for ZlibBase constructor')
 
@@ -439,3 +440,26 @@ export class BrotliDecompress extends Brotli {
     super(opts, 'BrotliDecompress')
   }
 }
+
+export class Zstd extends ZlibBase {
+  constructor(opts: ZlibOptions, mode: ZstdMode) {
+    opts = opts || {}
+
+    opts.flush = opts.flush || constants.ZSTD_e_continue
+    opts.finishFlush = opts.finishFlush || constants.ZSTD_e_end
+    opts.fullFlushFlag = constants.ZSTD_e_flush
+    super(opts, mode)
+  }
+}
+
+export class ZstdCompress extends Zstd {
+  constructor(opts: ZlibOptions) {
+    super(opts, 'ZstdCompress')
+  }
+}
+
+export class ZstdDecompress extends Zstd {
+  constructor(opts: ZlibOptions) {
+    super(opts, 'ZstdDecompress')
+  }
+}

src/index.ts

@@ -0,0 +1,23 @@
+'use strict'
+import t from 'tap'
+import { ZstdCompress } from '../dist/esm/index.js'
+import { readFileSync } from 'fs'
+import { fileURLToPath } from 'url'
+import * as console from 'console'
+
+const fixture = fileURLToPath(
+  new URL('fixtures/person.jpg', import.meta.url),
+)
+const file = readFileSync(fixture)
+const chunkSize = 16
+const deflater = new ZstdCompress()
+
+const chunk = file.subarray(0, chunkSize)
+const expectedFull = Buffer.from('KLUv/QBYgAAA/9j/4AAQSkZJRgABAQEASA==', 'base64')
+
+deflater.write(chunk)
+deflater.flush()
+const bufs = []
+deflater.on('data', b => bufs.push(b))
+const actualFull = Buffer.concat(bufs)
+t.same(actualFull, expectedFull)

test/fixtures/person.jpg.zst

@@ -0,0 +1,51 @@
+'use strict'
+// Test compressing and uncompressing a string with zstd
+
+import t from 'tap'
+import {ZstdCompress, ZstdDecompress} from '../dist/esm/index.js'
+
+const inputString =
+  'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' +
+  't. Morbi faucibus, purus at gravida dictum, libero arcu ' +
+  'convallis lacus, in commodo libero metus eu nisi. Nullam' +
+  ' commodo, neque nec porta placerat, nisi est fermentum a' +
+  'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' +
+  'n non diam orci. Proin quis elit turpis. Suspendisse non' +
+  ' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' +
+  'm arcu mi, sodales non suscipit id, ultrices ut massa. S' +
+  'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. '
+const compressedString =
+  'KLUv/WT5AF0JAGbXPCCgJUkH/8+rqgA3KaVsW+6LfK3JL' +
+  'cnP+I/Gy1/3Qv9XDTQAMwA0AK+Ch9LCub6tnT62C7Quwr' +
+  'HQHDhhNPcCQltMWOrafGy3KO2D79QZ95omy09vwp/TFEA' +
+  'kEIlHOO99cOlZmfRizXQ79GvDoY9TxrTgBBfR+77Nd7Lk' +
+  'OWlHaGW+aEwd2rSeegWaj9NsWAJJ0253u1jQpe3ByWLS5' +
+  'i+24QhTAZygaf4UlqNER3XoAk7QYar9tjHHV4yHj+tC10' +
+  '8zuqMBJ+X2hlpwUqX6vE3r3N7q5QYntVvn3N8zVDb9UfC' +
+  'MCW1790yV3A88pgvkvQAniSWvFxMAELvECFu0tC1R9Ijs' +
+  'ri5bt2kE/2mLoi2wCpkElnidDMS//DemxlNdHClyl6KeN' +
+  'TCugmAGfEYAXA=='
+
+t.test('compress then decompress', async t =>
+  new ZstdCompress()
+    .end(inputString)
+    .concat()
+    .then(async buffer => {
+      t.ok(
+        inputString.length > buffer.length,
+        'buffer is shorter than input',
+      )
+
+      return new ZstdDecompress()
+        .end(buffer)
+        .concat()
+        .then(buffer => t.equal(buffer.toString(), inputString))
+    }),
+)
+
+t.test('decompress then check', t =>
+  new ZstdDecompress({ encoding: 'utf8' })
+    .end(compressedString, 'base64')
+    .concat()
+    .then(result => t.equal(result, inputString)),
+)

test/zstd-flush.js

@@ -0,0 +1,35 @@
+'use strict'
+// Test unzipping a file that was created with a non-node zstd lib,
+// piped in as fast as possible.
+
+import t from 'tap'
+import { ZstdDecompress } from '../dist/esm/index.js'
+import fs from 'fs'
+import { resolve } from 'path'
+import { fileURLToPath } from 'url'
+
+const tmpdir = t.testdir({})
+
+const decompress = new ZstdDecompress()
+
+const fixture = fileURLToPath(
+  new URL('fixtures/person.jpg.zst', import.meta.url),
+)
+const unzippedFixture = fileURLToPath(
+  new URL('fixtures/person.jpg', import.meta.url),
+)
+const outputFile = resolve(tmpdir, 'person.jpg')
+const expect = fs.readFileSync(unzippedFixture)
+const inp = fs.createReadStream(fixture)
+const out = fs.createWriteStream(outputFile)
+
+t.test('decompress and test output', t => {
+  inp
+    .pipe(decompress)
+    .pipe(out)
+    .on('close', () => {
+      const actual = fs.readFileSync(outputFile)
+      t.same(actual, expect)
+      t.end()
+    })
+})

test/zstd-from-string.js

@@ -0,0 +1,79 @@
+import t from 'tap'
+import { constants, ZstdCompress } from '../dist/esm/index.js'
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+
+const fixture = fileURLToPath(
+  new URL('fixtures/pss-vectors.json', import.meta.url),
+)
+const sampleBuffer = fs.readFileSync(fixture)
+
+// Test some zstd-specific properties of the zstd streams that can not
+// be easily covered through expanding zlib-only tests.
+
+t.test('set compression level at stream creation', t => {
+  // Test setting the compression level at stream creation:
+  const sizes = []
+  for (
+    let quality = 0;
+    quality <= 5;
+    quality++
+  ) {
+    const encoded = new ZstdCompress({
+      params: {
+        [constants.ZSTD_c_compressionLevel]: quality,
+      },
+    })
+      .end(sampleBuffer)
+      .read()
+    sizes.push(encoded.length)
+  }
+
+  // Increasing quality should roughly correspond to decreasing compressed size:
+  for (let i = 0; i < sizes.length - 1; i++) {
+    t.ok(
+      sizes[i + 1] <= sizes[i] * 1.05,
+      `size ${i + 1} should be smaller than size ${i}`,
+    ) // 5 % margin of error.
+  }
+  t.ok(sizes[0] > sizes[sizes.length - 1], 'first size larger than last')
+
+  t.end()
+})
+
+t.test('setting out of bound option valules or keys fails', t => {
+  // Test that setting out-of-bounds option values or keys fails.
+  t.throws(
+    () => {
+      new ZstdCompress({
+        params: {
+          10000: 0,
+        },
+      })
+    },
+    {
+      code: 'ERR_ZSTD_INVALID_PARAM',
+      errno: undefined,
+      name: "ZlibError",
+    },
+  )
+
+  // Test that accidentally using duplicate keys fails.
+  t.throws(
+    () => {
+      new ZstdCompress({
+        params: {
+          0: 0,
+          '00': 0,
+        },
+      })
+    },
+    {
+      code: 'ERR_ZSTD_INVALID_PARAM',
+      errno: undefined,
+      name: "ZlibError",
+    },
+  )
+
+  t.end()
+})