Skip to content

Commit 9c3ead9

Browse files
iczeliaMintsuki
authored andcommitted
decompressor: gzip/tinf -> limlz
removes external dependency on tinf by replacing the compression algorithm with a simpler, faster, smaller and more auditable fixed-width LZ77 encoding purpose-tailored to x86 code mixed with data. before: decompressor.bin 2,492 bytes (tinf dependency) with .text 0x875 and .rodata 0x13c bytes each. after: decompressor.bin consists only of .text, 0xe6-byte decompressor; 90.8% reduction in decompressor volume. the dependency on gzip during compile-time is replaced by host/limlzpack.c, a Lempel-Ziv encoder in 275 SLoC that uses a suffix array matchfinder (prefix-doubling in mathcal O(n log^2 n) and Storer-Szymanski backwards parse. the fixed-width formats packets as [F][LLLL][MMM], favouring a literal-skewed distribution with F switching between one-byte and two-byte offsets (favouring recent statistics). integrity checking is done via crc32 with the polynomial 0xEDB88320, reflected. the effective loss in compression ratio by using a tremendously simpler and less packed with edge cases algorithm causes a compression ratio hit well below 1KB, factoring in the stub sizes. also adds new machinery for host cc detection per review.
1 parent 58083a7 commit 9c3ead9

File tree

18 files changed

+654
-199
lines changed

18 files changed

+654
-199
lines changed

.gitignore

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*.exe
1313
*.EFI
1414
*.bin
15-
*.bin.gz
15+
*.bin.limlz
1616
*.tar*
1717
*.elf
1818
*.hdd
@@ -31,10 +31,8 @@
3131
/common/lib/stb_image.h
3232
/common/cc-runtime.s2.c
3333
/cc-runtime
34-
/decompressor/tinf
3534
/decompressor/cc-runtime.c
3635
/libfdt
37-
/tinf
3836
/edk2-ovmf
3937
/bochsout.txt
4038
/bx_enh_dbg.ini

3RDPARTY.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,6 @@ below) provides headers and build-time support for UEFI.
4949
in case of installed copies, assuming the file has not been otherwise
5050
removed by the packager.
5151

52-
- [tinf](https://github.com/jibsen/tinf) (Zlib) is used in early x86 BIOS
53-
stages for GZIP decompression of stage2.
54-
5552
- [Flanterm](https://github.com/Mintsuki/Flanterm) (BSD-2-Clause) is used for
5653
text related screen drawing.
5754

GNUmakefile.in

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ AWK := @AWK@
5454
export AWK
5555

5656
CC := @CC@
57+
CC_FOR_BUILD := @CC_FOR_BUILD@
58+
CFLAGS_FOR_BUILD := @CFLAGS_FOR_BUILD@
5759

5860
CPPFLAGS := @CPPFLAGS@
5961
CFLAGS := @CFLAGS@
@@ -185,7 +187,7 @@ uninstall:
185187
rm -f '$(call SHESCAPE,$(DESTDIR)$(bindir))/limine'
186188
rm -rf '$(call SHESCAPE,$(DESTDIR)$(datarootdir))/limine'
187189

188-
$(call MKESCAPE,$(BUILDDIR))/stage1.stamp: $(STAGE1_FILES) $(call MKESCAPE,$(BUILDDIR))/decompressor-build/decompressor.bin $(call MKESCAPE,$(BUILDDIR))/common-bios/stage2.bin.gz
190+
$(call MKESCAPE,$(BUILDDIR))/stage1.stamp: $(STAGE1_FILES) $(call MKESCAPE,$(BUILDDIR))/decompressor-build/decompressor.bin $(call MKESCAPE,$(BUILDDIR))/common-bios/stage2.bin.limlz
189191
$(MKDIR_P) '$(call SHESCAPE,$(BINDIR))'
190192
cd '$(call SHESCAPE,$(SRCDIR))/stage1/hdd' && nasm bootsect.asm -Wall -w-unknown-warning -w-reloc $(WERROR_FLAG) -fbin -DBUILDDIR="'"'$(call NASMESCAPE,$(BUILDDIR))'"'" -o '$(call SHESCAPE,$(BINDIR))/limine-bios-hdd.bin'
191193
ifneq ($(BUILD_BIOS_CD),no)
@@ -301,7 +303,6 @@ dist:
301303
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/picoefi/.gitignore"
302304
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/cc-runtime"
303305
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/libfdt/.git"
304-
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/tinf"
305306
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/common/lib/stb_image.h.nopatch"
306307
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/.git"
307308
rm -rf '$(call SHESCAPE,$(BUILDDIR))'/"$(DIST_OUTPUT)/.gitignore"
@@ -327,7 +328,7 @@ distclean: clean
327328

328329
.PHONY: maintainer-clean
329330
maintainer-clean: distclean
330-
cd '$(call SHESCAPE,$(SRCDIR))' && rm -rf flanterm common/lib/stb_image.h.nopatch common/lib/stb_image.h decompressor/tinf tinf libfdt freestnd-c-hdrs cc-runtime common/cc-runtime.s2.c decompressor/cc-runtime.c limine-protocol picoefi configure timestamps build-aux *'~' autom4te.cache aclocal.m4 *.tar*
331+
cd '$(call SHESCAPE,$(SRCDIR))' && rm -rf flanterm common/lib/stb_image.h.nopatch common/lib/stb_image.h libfdt freestnd-c-hdrs cc-runtime common/cc-runtime.s2.c decompressor/cc-runtime.c limine-protocol picoefi configure timestamps build-aux *'~' autom4te.cache aclocal.m4 *.tar*
331332

332333
.PHONY: common-uefi-x86-64
333334
common-uefi-x86-64:
@@ -380,15 +381,20 @@ common-uefi-ia32-clean:
380381
rm -rf '$(call SHESCAPE,$(BUILDDIR))/common-uefi-ia32'
381382

382383
.PHONY: common-bios
383-
common-bios:
384+
common-bios: $(call MKESCAPE,$(BINDIR))/limlzpack
384385
$(MAKE) -C '$(call SHESCAPE,$(SRCDIR))/common' -f common.mk \
385386
TARGET=bios \
386-
BUILDDIR='$(call SHESCAPE,$(BUILDDIR))/common-bios'
387+
BUILDDIR='$(call SHESCAPE,$(BUILDDIR))/common-bios' \
388+
LIMLZPACK='$(call SHESCAPE,$(BINDIR))/limlzpack'
387389

388390
.PHONY: common-bios-clean
389391
common-bios-clean:
390392
rm -rf '$(call SHESCAPE,$(BUILDDIR))/common-bios'
391393

394+
$(call MKESCAPE,$(BINDIR))/limlzpack: $(call MKESCAPE,$(SRCDIR))/tools/limlzpack.c
395+
$(MKDIR_P) '$(call SHESCAPE,$(BINDIR))'
396+
$(CC_FOR_BUILD) $(CFLAGS_FOR_BUILD) -std=c99 -Wall -Wextra $(WERROR_FLAG) '$(call SHESCAPE,$<)' -o '$(call SHESCAPE,$@)'
397+
392398
.PHONY: decompressor
393399
decompressor:
394400
$(MAKE) -C '$(call SHESCAPE,$(SRCDIR))/decompressor' -f decompressor.mk \

INSTALL.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
In order to build Limine, the following programs have to be installed:
99
common UNIX tools (also known as `coreutils`),
10-
`GNU make`, `grep`, `sed`, `find`, `awk`, `gzip`, `nasm`, `mtools`
10+
`GNU make`, `grep`, `sed`, `find`, `awk`, `nasm`, `mtools`
1111
(optional, necessary to build `limine-uefi-cd.bin`).
1212
Furthermore, `gcc` or `llvm/clang` must also be installed, alongside
1313
the respective binutils.

bootstrap

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,15 +92,6 @@ if ! test -f version; then
9292
picoefi \
9393
9c99f66e4c15aebfd72e7becb72358473b484fcf
9494

95-
clone_repo_commit \
96-
https://github.com/jibsen/tinf.git \
97-
tinf \
98-
57ffa1f1d5e3dde19011b2127bd26d01689b694b
99-
mkdir -p decompressor/tinf
100-
cp tinf/src/tinf.h tinf/src/tinflate.c tinf/src/tinfgzip.c tinf/src/crc32.c decompressor/tinf/
101-
patch -p0 < decompressor/tinf.patch
102-
rm -f decompressor/tinf/*.orig
103-
10495
clone_repo_commit \
10596
https://github.com/Mintsuki/Flanterm.git \
10697
flanterm \

common/common.mk

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ override HEADER_DEPS := $(addprefix $(call MKESCAPE,$(BUILDDIR))/, $(C_FILES:.c=
304304
.PHONY: all
305305

306306
ifeq ($(TARGET),bios)
307-
all: $(call MKESCAPE,$(BUILDDIR))/limine-bios.sys $(call MKESCAPE,$(BUILDDIR))/stage2.bin.gz
307+
all: $(call MKESCAPE,$(BUILDDIR))/limine-bios.sys $(call MKESCAPE,$(BUILDDIR))/stage2.bin.limlz
308308
endif
309309
ifeq ($(TARGET),uefi-x86-64)
310310
all: $(call MKESCAPE,$(BUILDDIR))/BOOTX64.EFI
@@ -324,8 +324,8 @@ endif
324324

325325
ifeq ($(TARGET),bios)
326326

327-
$(call MKESCAPE,$(BUILDDIR))/stage2.bin.gz: $(call MKESCAPE,$(BUILDDIR))/stage2.bin
328-
gzip -n -9 < '$(call SHESCAPE,$<)' > '$(call SHESCAPE,$@)'
327+
$(call MKESCAPE,$(BUILDDIR))/stage2.bin.limlz: $(call MKESCAPE,$(BUILDDIR))/stage2.bin $(LIMLZPACK)
328+
'$(call SHESCAPE,$(LIMLZPACK))' '$(call SHESCAPE,$<)' '$(call SHESCAPE,$@)'
329329

330330
$(call MKESCAPE,$(BUILDDIR))/stage2.bin: $(call MKESCAPE,$(BUILDDIR))/limine-bios.sys
331331
dd if='$(call SHESCAPE,$<)' bs=$$(( 0x$$("$(READELF_FOR_TARGET)" -S '$(call SHESCAPE,$(BUILDDIR))/limine.elf' | $(GREP) '\.text\.stage3' | $(SED) 's/^.*] //' | $(AWK) '{print $$3}' | $(SED) 's/^0*//') - 0xf000 )) count=1 of='$(call SHESCAPE,$@)' 2>/dev/null

configure.ac

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ AC_INIT([Limine], [m4_esyscmd([./version.sh])], [https://github.com/Limine-Bootl
33
AC_PREREQ([2.69])
44

55
AC_CONFIG_AUX_DIR([build-aux])
6+
AC_CONFIG_MACRO_DIRS([m4])
67

78
SRCDIR="$(cd "$srcdir" && pwd -P)"
89
BUILDDIR="$(pwd -P)"
@@ -17,6 +18,7 @@ AC_SUBST([SOURCE_DATE_EPOCH])
1718
AC_SUBST([SOURCE_DATE_EPOCH_TOUCH])
1819

1920
AC_CANONICAL_HOST
21+
AC_CANONICAL_BUILD
2022

2123
# Portably convert relative paths into absolute paths.
2224
rel2abs() {
@@ -52,6 +54,9 @@ AC_LANG([C])
5254
AC_PROG_CC
5355
CC="$(rel2abs "$CC")"
5456

57+
AX_PROG_CC_FOR_BUILD
58+
CC_FOR_BUILD="$(rel2abs "$CC_FOR_BUILD")"
59+
5560
werror_state="no"
5661
AC_ARG_ENABLE([werror],
5762
[AS_HELP_STRING([--enable-werror], [treat warnings as errors])],
@@ -208,7 +213,6 @@ if test "x$BUILD_BIOS" = "xno"; then
208213
else
209214
BUILD_BIOS="limine-bios"
210215
NEED_NASM=yes
211-
NEED_GZIP=yes
212216
fi
213217

214218
AC_SUBST([BUILD_BIOS])
@@ -311,12 +315,6 @@ if test "x$NEED_NASM" = "xyes"; then
311315
fi
312316
fi
313317

314-
if test "x$NEED_GZIP" = "xyes"; then
315-
AC_CHECK_PROG([GZIP_FOUND], [gzip], [yes])
316-
if ! test "x$GZIP_FOUND" = "xyes"; then
317-
AC_MSG_ERROR([gzip not found, please install gzip before configuring])
318-
fi
319-
fi
320318

321319
BORROWED_CFLAGS=""
322320
for cflag in $CFLAGS; do

decompressor/decompressor.asm

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
; limlz: Copyright (C) 2026 Kamila Szewczyk <k@iczelia.net>
2+
; limine: Copyright (C) 2019-2026 Mintsuki and contributors.
3+
;
4+
; Redistribution and use in source and binary forms, with or without
5+
; modification, are permitted provided that the following conditions are met:
6+
;
7+
; 1. Redistributions of source code must retain the above copyright notice, this
8+
; list of conditions and the following disclaimer.
9+
;
10+
; 2. Redistributions in binary form must reproduce the above copyright notice,
11+
; this list of conditions and the following disclaimer in the documentation
12+
; and/or other materials provided with the distribution.
13+
;
14+
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15+
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16+
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17+
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
18+
; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19+
; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20+
; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
21+
; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22+
; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23+
; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24+
25+
bits 32
26+
27+
section .entry progbits alloc exec nowrite align=16
28+
29+
global _start
30+
_start:
31+
cld
32+
; On stack (cdecl): [esp+4]=compressed_stage2, [esp+8]=stage2_size,
33+
; [esp+12]=boot_drive (byte), [esp+16]=pxe
34+
mov ebx, dword [esp+0x4] ; compressed_stage2
35+
mov ebp, dword [ebx] ; expected_crc = *(uint32_t *)compressed_stage2
36+
lea edx, [ebx+0x4] ; ip = compressed_stage2 + 4
37+
add ebx, dword [esp+0x8] ; ipe = compressed_stage2 + stage2_size
38+
mov edi, 0xf000 ; op = dest
39+
; LZ decompression loop
40+
.Ltoken:
41+
movzx ecx, byte [edx]
42+
lea esi, [edx+0x1]
43+
mov eax, ecx ; save token
44+
shr ecx, 0x3
45+
and ecx, 0xf ; literal length = (token >> 3) & 15
46+
cmp ecx, 0xf
47+
jne .Llitcopy
48+
movzx ecx, byte [edx+0x1]
49+
lea esi, [edx+0x2]
50+
add ecx, 0xf ; length += extra byte + 15
51+
.Llitcopy:
52+
rep movsb ; copy literals
53+
cmp esi, ebx
54+
jae .Lcrc ; if ip >= ipe, done
55+
test al, al
56+
jns .Loffset1 ; bit 7 clear => 1-byte offset
57+
lea edx, [esi+0x2]
58+
movzx esi, word [esi] ; 2-byte offset
59+
jmp .Lmatchlen
60+
.Loffset1:
61+
lea edx, [esi+0x1]
62+
movzx esi, byte [esi] ; 1-byte offset
63+
.Lmatchlen:
64+
and al, 0x7
65+
cmp al, 0x7
66+
je .Lmatchextra
67+
movzx eax, al
68+
jmp .Ldomatch
69+
.Lmatchextra:
70+
movzx eax, byte [edx]
71+
inc edx
72+
add eax, 0x7 ; matchlen += extra byte + 7
73+
.Ldomatch:
74+
mov ecx, edi
75+
sub ecx, esi ; match = op - offset
76+
mov esi, ecx
77+
lea ecx, [eax+0x4] ; count = matchlen + 4
78+
rep movsb ; copy match
79+
jmp .Ltoken
80+
; CRC32 verification
81+
.Lcrc:
82+
mov edx, 0xf000 ; ptr = dest
83+
mov esi, edx ; (also reused for esp later)
84+
xor eax, eax
85+
dec eax
86+
.Lcrc_byte:
87+
cmp edx, edi
88+
je .Lcrc_done
89+
lea ecx, [edx+0x1]
90+
movzx edx, byte [edx]
91+
xor eax, edx
92+
push 0x08
93+
pop edx ; 8 bits per byte
94+
.Lcrc_bit:
95+
mov ebx, eax
96+
and eax, 0x1
97+
shr ebx, 1
98+
neg eax
99+
and eax, 0xedb88320
100+
xor eax, ebx
101+
dec edx
102+
jne .Lcrc_bit
103+
mov edx, ecx
104+
jmp .Lcrc_byte
105+
.Lcrc_done:
106+
not eax
107+
cmp eax, ebp
108+
jne .Lerror
109+
; Jump to decompressed stage2
110+
movzx eax, byte [esp+0xc] ; boot_drive
111+
mov ecx, dword [esp+0x10] ; pxe
112+
mov esp, esi
113+
xor ebp, ebp
114+
push ecx
115+
push eax
116+
push ebp
117+
push esi
118+
ret ; jump to 0xf000
119+
; Error: display message and cli/hlt
120+
.Lerror:
121+
mov edx, errmsg
122+
mov eax, 0xb8000
123+
.Lerror_loop:
124+
movzx ecx, byte [edx]
125+
add eax, 0x2
126+
inc edx
127+
or ch, 0x4f
128+
mov word [eax-0x2], cx
129+
cmp eax, 0xB8000 + errmsg.len * 2
130+
jne .Lerror_loop
131+
cli
132+
hlt
133+
134+
section .rodata progbits alloc noexec nowrite align=1
135+
136+
errmsg: db "limine integrity error"
137+
.len: equ $ - errmsg - 1
138+
139+
section .note.GNU-stack noalloc noexec nowrite progbits

decompressor/decompressor.mk

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ override CFLAGS_FOR_TARGET += \
3939

4040
override CPPFLAGS_FOR_TARGET := \
4141
-I . \
42-
-I tinf \
4342
-isystem ../freestnd-c-hdrs/include \
4443
$(CPPFLAGS_FOR_TARGET) \
4544
-MMD \

decompressor/entry.asm

Lines changed: 0 additions & 20 deletions
This file was deleted.

0 commit comments

Comments
 (0)