Skip to content

Commit e9a95e0

Browse files
authored
compress: transparent gzip-compression (#545) (#549)
* compress: transparent gzip-compression provides a port of embeddable pdgzip (CC0) with small performance tweaks to attain better-than-zlib decode times. we have a couple of unique tricks up our sleeve: - extra validation of huffman trees via the kraft-mcmillan condition - multi-level huffman tables; standard technique in accelerated zlib decompressors, here implemented in a low code volume. - semi-space based window design to decrease overall latency and move a hotspot to semi-space memcpy() back to history buffer. - full support of the RFC quirks for deflate/gzip, including fixed huffman trees. - fast crc32 implementation via runtime-computed tables and the slicing-by-4 algorithm. comparison (size): tinf ~2.5kb x86 code, this ~15kb x86 code, zlib ~22kb of x86 code. comparison (code volume): tinf 639 sloc, this 594 sloc, zlib >=10k sloc, libdeflate >=7.7k sloc. performance (r7 pro 7840u; enwik8 100MB): tinf ~2.3s, this ~409.7 ms, zlib ~417.7 ms. extra: fuzzed with afl++. pending addition of fuzzing scripts and automated ci-bound testing. thin wrapper over file streams automatically used when the $-prefix is found as per #545. * contrib: check in the gzip-fuzzing harness this fuzzing harness is not intended to be ran by end users or distro maintainers, hence it does not follow a lot of the standard "portability" kludges of mainline limine code. * compress: gzip.c/.h - document limitations. document the behaviour of the ->size field in the public API of gzip.h; adhere to a more uniform style. * fix ci replace the 2M image with a 32M UEFI image with a specific head/track/sector geometry. * test: limine.c now uses outw() for QEMU-specific fast path shutdown. * gzip.c: more memory-frugal by constructing fixed tables on demand also move crc32 tables to ext_mem_alloc, document peak mem usage * contrib: mechanical test for validating gzip compression * minor cosmetic * compress: properly hook up gzip to uri_open - remove ISIZE parsing in gzip.c - add a streaming file_handle wrapper over blake2b. - make ->read and fread return the actual # of bytes read (needed by the gzip decoder downstream users to determine real EOF) - add is_high_mem and load_addr_64 to file_handle (after freadall_mode inlined body in uri_open relocates to high memory) - as a result of the changes, freadall and freadall_mode are no longer used anywhere by the code base. hence they were removed. - update the signature of uri_open to accept memcpy functions to/from high memory and a boolean parameter for whether high memory allocations are acceptable for this specific resource. - inline size-agnostic (stretchy-vector type) functionality to uri_open to facilitate streaming unknown-size decoding. uri_open now returns memfiles, always, mimicking the behaviour prior to streaming blake2b change commit. - minimum patch in limine_asm to make limine_memcpy_64_asm take two wide pointers. - update downstream callers of uri_open, as well as the gzip fuzzing suite. * fs: fat32.s2.c: update a stale comment * crypt: blake2b.h: opaque pointer to fs guts * lib: uri.c silence warning on non-i386 * lib: uri.c: remove stale comment. * move to pdgzip * drop old * test/test.mk: rollback a merge artifact. * test.mk: re-add extra cflags
1 parent 35f76ad commit e9a95e0

26 files changed

+793
-180
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
/common/lib/stb_image.h
3232
/common/cc-runtime.s2.c
3333
/cc-runtime
34+
/common/compress/pdgzip.c
35+
/common/compress/pdgzip.h
36+
/pdgzip
3437
/libfdt
3538
/edk2-ovmf
3639
/bochsout.txt

3RDPARTY.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ used for wallpaper image loading.
5858
- [libfdt](https://github.com/osdev0/libfdt) (BSD-2-Clause) is used for
5959
manipulating Flat Device Trees.
6060

61+
- [pdgzip](https://github.com/iczelia/pdgzip) (0BSD) is used to provide the
62+
transparent gzip decompression layer for loaded files.
63+
6164
Note that some of these projects, or parts of them, are provided under
6265
dual-licensing, in which case, in the above list, the only license mentioned is
6366
the one chosen by the Limine developers. Refer to each individual project's

GNUmakefile.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,8 @@ $(call MKESCAPE,$(BINDIR))/limine-uefi-cd.bin: $(if $(BUILD_UEFI_IA32),$(call MK
209209
ifneq ($(BUILD_UEFI_CD),no)
210210
$(MKDIR_P) '$(call SHESCAPE,$(BINDIR))'
211211
rm -f '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin'
212-
dd if=/dev/zero of='$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' bs=512 count=5760 2>/dev/null
213-
mformat -i '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' -f 2880 -N 12345678 ::
212+
dd if=/dev/zero of='$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' bs=512 count=32768 2>/dev/null
213+
mformat -i '$(call SHESCAPE,$(BINDIR))/limine-uefi-cd.bin' -h 64 -t 32 -s 16 -N 12345678 ::
214214
LIMINE_UEFI_CD_TMP="$$(mktemp -d)"; \
215215
mkdir -p "$$LIMINE_UEFI_CD_TMP"/EFI/BOOT; \
216216
cp '$(call SHESCAPE,$(BUILDDIR))/common-uefi-aarch64/BOOTAA64.EFI' "$$LIMINE_UEFI_CD_TMP"/EFI/BOOT/ 2>/dev/null; \

bootstrap

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ if ! test -f version; then
8181
dae79833b57a01b9fd3e359ee31def69f5ae899b
8282
cp cc-runtime/src/cc-runtime.c common/cc-runtime.s2.c
8383

84+
clone_repo_commit \
85+
https://github.com/iczelia/pdgzip.git \
86+
pdgzip \
87+
16c41d9af067c4185c136622c58ad4188609a3d1
88+
cp pdgzip/pdgzip.c common/compress/pdgzip.c
89+
cp pdgzip/pdgzip.h common/compress/pdgzip.h
90+
8491
clone_repo_commit \
8592
https://github.com/Limine-Bootloader/limine-protocol.git \
8693
limine-protocol \

common/compress/gzip.c

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/* Limine glue around pdgzip: transparent gzip decompression layer over a
2+
* file_handle. The underlying decoder lives in common/compress/pdgzip.c
3+
* (imported by ./bootstrap from the upstream iczelia/pdgzip repo); this
4+
* file only wires pdgzip's streaming read-callback API into Limine's
5+
* file_handle abstraction and adds support for random-access reads via
6+
* rewind-and-skip.
7+
*
8+
* Copyright (C) 2019-2026 Mintsuki and contributors.
9+
*
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice, this
14+
* list of conditions and the following disclaimer.
15+
*
16+
* 2. Redistributions in binary form must reproduce the above copyright notice,
17+
* this list of conditions and the following disclaimer in the documentation
18+
* and/or other materials provided with the distribution.
19+
*
20+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21+
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22+
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23+
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24+
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25+
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26+
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27+
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28+
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29+
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
*/
31+
32+
#include <stdint.h>
33+
#include <stddef.h>
34+
#include <stdbool.h>
35+
#include <lib/libc.h>
36+
#include <lib/misc.h>
37+
#include <lib/print.h>
38+
#include <mm/pmm.h>
39+
#include <compress/gzip.h>
40+
#include <compress/pdgzip.h>
41+
42+
struct gzip_handle {
43+
struct file_handle * source; /* compressed file (owned) */
44+
pdgzip_t * gz; /* decoder backed by `scratch` */
45+
void * scratch; /* pdgzip scratch buffer */
46+
size_t scratch_sz;
47+
uint64_t src_pos; /* next byte to pull from `source` */
48+
uint64_t dec_pos; /* current decompressed stream offset */
49+
};
50+
51+
/* pdgzip read callback: pull up to `len` bytes from the compressed source
52+
starting at gh->src_pos. A short read (including zero) signals EOF to
53+
the decoder, which is correct at the end of the file. */
54+
static size_t gz_source_read(void * user, void * buf, size_t len) {
55+
struct gzip_handle * gh = user;
56+
uint64_t avail = gh->source->size - gh->src_pos;
57+
if ((uint64_t)len > avail) len = (size_t)avail;
58+
if (len == 0) return 0;
59+
fread(gh->source, buf, gh->src_pos, len);
60+
gh->src_pos += len;
61+
return len;
62+
}
63+
64+
/* (Re)initialize the decoder for a fresh pass over the compressed stream.
65+
pdgzip_init zeroes its own scratch, so we only need to reset our own
66+
bookkeeping. */
67+
static void gz_reset(struct gzip_handle * gh) {
68+
pdgzip_cfg_t cfg = { .read = gz_source_read, .user = gh, .concat = 0 };
69+
gh->src_pos = 0;
70+
gh->dec_pos = 0;
71+
gh->gz = pdgzip_init(gh->scratch, &cfg);
72+
}
73+
74+
static uint64_t gzip_read(struct file_handle * file, void * buf, uint64_t loc, uint64_t count) {
75+
struct gzip_handle * gh = file->fd;
76+
/* Rewind on backward seeks. */
77+
if (loc < gh->dec_pos) gz_reset(gh);
78+
/* Skip forward to reach the requested offset. EOS during seek means
79+
the requested location is past end-of-stream - return 0 bytes. */
80+
while (gh->dec_pos < loc) {
81+
uint8_t discard[4096];
82+
uint64_t gap = loc - gh->dec_pos;
83+
size_t chunk = gap > sizeof(discard) ? sizeof(discard) : (size_t)gap;
84+
int64_t n = pdgzip_read(gh->gz, discard, chunk);
85+
if (n < 0) panic(false, "gzip: decompression error during seek");
86+
if (n == 0) return 0;
87+
gh->dec_pos += (uint64_t)n;
88+
}
89+
/* Decompress the requested data. */
90+
uint8_t * dst = buf;
91+
uint64_t remaining = count;
92+
while (remaining > 0) {
93+
size_t chunk = remaining > 65536 ? 65536 : (size_t)remaining;
94+
int64_t n = pdgzip_read(gh->gz, dst, chunk);
95+
if (n < 0) panic(false, "gzip: decompression error");
96+
if (n == 0) break;
97+
dst += n;
98+
remaining -= (uint64_t)n;
99+
gh->dec_pos += (uint64_t)n;
100+
}
101+
return count - remaining;
102+
}
103+
104+
static void gzip_close(struct file_handle * file) {
105+
struct gzip_handle * gh = file->fd;
106+
fclose(gh->source);
107+
pmm_free(gh->scratch, gh->scratch_sz);
108+
pmm_free(gh, sizeof(struct gzip_handle));
109+
}
110+
111+
bool gzip_check(struct file_handle * fd) {
112+
if (fd->size < 18) return false;
113+
uint8_t magic[2]; fread(fd, magic, 0, 2);
114+
return magic[0] == 0x1F && magic[1] == 0x8B;
115+
}
116+
117+
struct file_handle * gzip_open(struct file_handle * compressed) {
118+
/* The decompressed size is not known up front. The 4-byte ISIZE trailer
119+
is unreliable (modulo 2^32, spec defect) and callers must instead
120+
drain until gzip_read returns 0 bytes (EOS). Advertise an unknown
121+
size via UINT64_MAX. */
122+
struct gzip_handle * gh = ext_mem_alloc(sizeof(struct gzip_handle));
123+
gh->source = compressed;
124+
gh->scratch_sz = pdgzip_state_size();
125+
gh->scratch = ext_mem_alloc(gh->scratch_sz);
126+
gz_reset(gh);
127+
/* Depends on ext_mem_alloc returning zeroed memory. */
128+
struct file_handle * ret = ext_mem_alloc(sizeof(struct file_handle));
129+
ret->fd = gh;
130+
ret->read = (void *) gzip_read;
131+
ret->close = (void *) gzip_close;
132+
ret->size = UINT64_MAX;
133+
ret->vol = compressed->vol;
134+
if (compressed->path != NULL && compressed->path_len > 0) {
135+
ret->path = ext_mem_alloc(compressed->path_len);
136+
memcpy(ret->path, compressed->path, compressed->path_len);
137+
ret->path_len = compressed->path_len;
138+
}
139+
#if defined (UEFI)
140+
ret->efi_part_handle = compressed->efi_part_handle;
141+
#endif
142+
ret->pxe = compressed->pxe;
143+
ret->pxe_ip = compressed->pxe_ip;
144+
ret->pxe_port = compressed->pxe_port;
145+
return ret;
146+
}

common/compress/gzip.h

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/* embeddable gzip decoder: Copyright (C) 2026 Kamila Szewczyk <k@iczelia.net>
2+
* limine: Copyright (C) 2019-2026 Mintsuki and contributors.
3+
*
4+
* Redistribution and use in source and binary forms, with or without
5+
* modification, are permitted provided that the following conditions are met:
6+
*
7+
* 1. Redistributions of source code must retain the above copyright notice, this
8+
* list of conditions and the following disclaimer.
9+
*
10+
* 2. Redistributions in binary form must reproduce the above copyright notice,
11+
* this list of conditions and the following disclaimer in the documentation
12+
* and/or other materials provided with the distribution.
13+
*
14+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15+
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16+
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17+
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
18+
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19+
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20+
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
21+
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22+
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23+
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24+
*/
25+
26+
#ifndef COMPRESS__GZIP_H__
27+
#define COMPRESS__GZIP_H__
28+
29+
#include <fs/file.h>
30+
31+
/* Check if a file handle points to gzip-compressed data (0x1F 0x8B magic). */
32+
bool gzip_check(struct file_handle * fd);
33+
34+
/* Wrap a gzip-compressed file handle in a decompressing layer.
35+
*
36+
* Returns a new file_handle whose read callback transparently
37+
* decompresses the data. The returned handle takes ownership of
38+
* `compressed` and will close it when itself is closed.
39+
*
40+
* WARNING: Due to a Gzip format deficiency, ->size of the resulting
41+
* file_handle is only an approximation (i.e., it is not correct for
42+
* files larger than 4 GiB and doesn't necessarily have to reflect
43+
* the genuine decompressed size at all in adversarial circumstances).
44+
*
45+
* The real decompressed size can only be authoritatively obtained by
46+
* fully decompressing the file.
47+
*
48+
* Supports very fast sequential reads and random-access reads (with
49+
* an implicit rewind + skip penalty inherent to the gzip format).
50+
*/
51+
struct file_handle * gzip_open(struct file_handle * compressed);
52+
53+
#endif

common/crypt/blake2b.c

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
#include <stddef.h>
77
#include <crypt/blake2b.h>
88
#include <lib/libc.h>
9+
#include <lib/misc.h>
10+
#include <mm/pmm.h>
911

1012
#define BLAKE2B_BLOCK_BYTES 128
1113
#define BLAKE2B_KEY_BYTES 64
@@ -218,3 +220,73 @@ void blake2b(void *out, const void *in, size_t in_len) {
218220
blake2b_update(&state, in, in_len);
219221
blake2b_final(&state, out);
220222
}
223+
224+
/* Streaming filter: wraps a source file_handle and hashes bytes as
225+
they are read sequentially. The hash is finalized and compared via
226+
blake2b_check_hash(). Non-sequential reads panic -- the filter is
227+
meant to sit underneath the gzip bitreader or uri_open's drain loop,
228+
both of which advance monotonically. */
229+
struct blake2b_handle {
230+
struct file_handle *source;
231+
struct blake2b_state state;
232+
uint64_t pos;
233+
bool finalized;
234+
uint8_t digest[BLAKE2B_OUT_BYTES];
235+
};
236+
237+
static uint64_t blake2b_read(struct file_handle *fh, void *buf, uint64_t loc, uint64_t count) {
238+
struct blake2b_handle *h = fh->fd;
239+
if (loc != h->pos) {
240+
panic(false, "blake2b filter: non-sequential read (pos=%x, loc=%x)",
241+
(uint64_t)h->pos, loc);
242+
}
243+
uint64_t got = fread(h->source, buf, loc, count);
244+
blake2b_update(&h->state, buf, got);
245+
h->pos += got;
246+
return got;
247+
}
248+
249+
static void blake2b_close(struct file_handle *fh) {
250+
struct blake2b_handle *h = fh->fd;
251+
fclose(h->source);
252+
pmm_free(h, sizeof(struct blake2b_handle));
253+
}
254+
255+
struct file_handle *blake2b_open(struct file_handle *source) {
256+
struct blake2b_handle *h = ext_mem_alloc(sizeof(struct blake2b_handle));
257+
blake2b_init(&h->state);
258+
h->source = source;
259+
h->pos = 0;
260+
h->finalized = false;
261+
262+
struct file_handle *ret = ext_mem_alloc(sizeof(struct file_handle));
263+
ret->fd = h;
264+
ret->read = (void *)blake2b_read;
265+
ret->close = (void *)blake2b_close;
266+
ret->size = source->size;
267+
ret->vol = source->vol;
268+
if (source->path != NULL && source->path_len > 0) {
269+
ret->path = ext_mem_alloc(source->path_len);
270+
memcpy(ret->path, source->path, source->path_len);
271+
ret->path_len = source->path_len;
272+
}
273+
#if defined (UEFI)
274+
ret->efi_part_handle = source->efi_part_handle;
275+
#endif
276+
ret->pxe = source->pxe;
277+
ret->pxe_ip = source->pxe_ip;
278+
ret->pxe_port = source->pxe_port;
279+
return ret;
280+
}
281+
282+
bool blake2b_check_hash(struct file_handle *fh, void *reference_hash) {
283+
if (fh->read != (void *)blake2b_read) {
284+
panic(false, "blake2b_check_hash: not a blake2b filter handle");
285+
}
286+
struct blake2b_handle *h = fh->fd;
287+
if (!h->finalized) {
288+
blake2b_final(&h->state, h->digest);
289+
h->finalized = true;
290+
}
291+
return memcmp(h->digest, reference_hash, BLAKE2B_OUT_BYTES) == 0;
292+
}

common/crypt/blake2b.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,8 @@
77

88
void blake2b(void *out, const void *in, size_t in_len);
99

10+
struct file_handle;
11+
struct file_handle * blake2b_open(struct file_handle * source);
12+
bool blake2b_check_hash(struct file_handle *fd, void* reference_hash);
13+
1014
#endif

common/fs/fat32.s2.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -605,7 +605,7 @@ char *fat32_get_label(struct volume *part) {
605605
return context.label;
606606
}
607607

608-
static void fat32_read(struct file_handle *handle, void *buf, uint64_t loc, uint64_t count);
608+
static uint64_t fat32_read(struct file_handle *handle, void *buf, uint64_t loc, uint64_t count);
609609
static void fat32_close(struct file_handle *file);
610610

611611
struct file_handle *fat32_open(struct volume *part, const char *path) {
@@ -622,7 +622,7 @@ struct file_handle *fat32_open(struct volume *part, const char *path) {
622622
unsigned int current_index = 0;
623623
char current_part[FAT32_LFN_MAX_FILENAME_LENGTH];
624624

625-
// skip trailing slashes
625+
// skip leading slashes
626626
while (path[current_index] == '/') {
627627
current_index++;
628628
}
@@ -719,11 +719,12 @@ struct file_handle *fat32_open(struct volume *part, const char *path) {
719719
}
720720
}
721721

722-
static void fat32_read(struct file_handle *file, void *buf, uint64_t loc, uint64_t count) {
722+
static uint64_t fat32_read(struct file_handle *file, void *buf, uint64_t loc, uint64_t count) {
723723
struct fat32_file_handle *f = file->fd;
724724
if (!read_cluster_chain(&f->context, f->cluster_chain, f->chain_len, buf, loc, count)) {
725725
panic(false, "fat32: cluster chain read failed (corrupted filesystem?)");
726726
}
727+
return count;
727728
}
728729

729730
static void fat32_close(struct file_handle *file) {

0 commit comments

Comments
 (0)