Skip to content

Modify nginx compile script for dynamic compilation #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rishabhBudhouliya wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Modify nginx compile script for dynamic compilation #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e86a3f9
update lmbench compilation script for dynamic loading
rishabhBudhouliya Mar 30, 2026
4930659
Merge branch 'main' into dynamic-lmbench
rishabhBudhouliya Apr 4, 2026
314c751
add dynamic compilation support for nginx
rishabhBudhouliya Apr 4, 2026
668ef5e
update config path
rishabhBudhouliya Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 92 additions & 9 deletions lmbench/src/compile_lmbench.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ set -euo pipefail
# 3) Stage final artifacts under build/lmbench/bin.
# 4) Require cwasm generation for the staged executables and copy the
# resulting .cwasm files back onto the final extensionless program names.
#
# Dynamic loading support:
# Set LIND_DYLINK=1 to build lmbench as position-independent executables
# for the dylink branch. This adds -fPIC to CFLAGS, uses PIE link flags,
# and applies the dylink-aware wasm-opt passes.

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
Expand All @@ -32,6 +37,7 @@ JOBS="${JOBS:-$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN || echo 4)}"
# lmbench's staged executables now always require the full cwasm-backed output
# set mandated by issue #127, regardless of artifact mode.
##################
LIND_DYLINK="${LIND_DYLINK:-0}"
ARTIFACT_MODE="${ARTIFACT_MODE:-fast}"
FORCE_CLEAN="${FORCE_CLEAN:-0}"
FORCE_CONFIGURE="${FORCE_CONFIGURE:-0}"
Expand Down Expand Up @@ -118,11 +124,28 @@ run_wasm_opt_replace() {
local out="$3"
local tmp="${out}.tmp"
cp "$src" "$raw"
if "$WASM_OPT" --fpcast-emu --epoch-injection --asyncify --debuginfo -O2 "$raw" -o "$tmp"; then
mv "$tmp" "$out"
if [[ "$LIND_DYLINK" == "1" ]]; then
# dylink-aware wasm-opt: epoch-import and asyncify-import-globals
# are needed because epoch counter and asyncify globals live in the
# host (libc module), not in the main module.
if "$WASM_OPT" \
--enable-bulk-memory --enable-threads \
--epoch-injection --pass-arg=epoch-import --pass-arg=epoch-main-module \
--asyncify --pass-arg=asyncify-import-globals \
--debuginfo \
"$raw" -o "$tmp"; then
mv "$tmp" "$out"
else
rm -f "$tmp"
return 1
fi
else
rm -f "$tmp"
return 1
if "$WASM_OPT" --fpcast-emu --epoch-injection --asyncify --debuginfo -O2 "$raw" -o "$tmp"; then
mv "$tmp" "$out"
else
rm -f "$tmp"
return 1
fi
fi
}

Expand Down Expand Up @@ -171,11 +194,52 @@ mkdir -p "$LM_BENCH_BIN_DIR"

REAL_CC="$CLANG --target=wasm32-unknown-wasi --sysroot=$MERGED_SYSROOT"
CFLAGS="-DNO_PORTMAPPER -O2 -g -I$MERGED_SYSROOT/include -I$MERGED_SYSROOT/include/wasm32-wasi -I$MERGED_SYSROOT/include/tirpc"
LDFLAGS_WASM=(
"-Wl,--import-memory,--export-memory,--max-memory=${MAX_WASM_MEMORY},--export=__stack_pointer,--export=__stack_low,--export=__tls_base"
"-L$MERGED_SYSROOT/lib/wasm32-wasi"
"-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
)

if [[ "$LIND_DYLINK" == "1" ]]; then
echo "[lmbench] Dynamic linking mode enabled (LIND_DYLINK=1)"
CFLAGS+=" -fPIC"
# add-export-tool is used after linking to export relocation and stack symbols
ADD_EXPORT_TOOL="$LIND_WASM_ROOT/tools/add-export-tool/add-export-tool"
if [[ ! -x "$ADD_EXPORT_TOOL" ]]; then
echo "[lmbench] ERROR: add-export-tool not found at '$ADD_EXPORT_TOOL'" >&2
exit 1
fi
# Extra objects required for dynamic PIE executables
DYLINK_CRT_OBJS=(
"$MERGED_SYSROOT/lib/wasm32-wasi/set_stack_pointer.o"
"$MERGED_SYSROOT/lib/wasm32-wasi/crt1_shared.o"
"$MERGED_SYSROOT/lib/wasm32-wasi/lind_utils.o"
)
for obj in "${DYLINK_CRT_OBJS[@]}"; do
if [[ ! -f "$obj" ]]; then
echo "[lmbench] ERROR: required dylink CRT object '$obj' not found." >&2
echo "[lmbench] Hint: rebuild sysroot on the dylink branch (make sysroot in lind-wasm)." >&2
exit 1
fi
done
LDFLAGS_WASM=(
"-nostartfiles"
"-Wl,-pie"
"-Wl,--import-table"
"-Wl,--import-memory"
"-Wl,--export-memory"
"-Wl,--max-memory=${MAX_WASM_MEMORY}"
"-Wl,--allow-undefined"
"-Wl,--unresolved-symbols=import-dynamic"
"-Wl,--export=__wasm_call_ctors"
"-Wl,--export-if-defined=__wasm_init_tls"
"-Wl,--export=__tls_base"
"-L$MERGED_SYSROOT/lib/wasm32-wasi"
"-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
"${DYLINK_CRT_OBJS[@]}"
)
else
LDFLAGS_WASM=(
"-Wl,--import-memory,--export-memory,--max-memory=${MAX_WASM_MEMORY},--export=__stack_pointer,--export=__stack_low,--export=__tls_base"
"-L$MERGED_SYSROOT/lib/wasm32-wasi"
"-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
)
fi

if [[ "$ENABLE_WASI_THREADS" == "1" ]]; then
thread_flag="-mthread-model=posix"
Expand Down Expand Up @@ -212,6 +276,7 @@ build_signature() {
echo "merged=$MERGED_SYSROOT"
echo "memory=$MAX_WASM_MEMORY"
echo "threads=$ENABLE_WASI_THREADS"
echo "dylink=$LIND_DYLINK"
echo "cflags=$CFLAGS"
echo "ldflags=$LDFLAGS"
echo "ldlibs=$LDLIBS"
Expand Down Expand Up @@ -367,6 +432,24 @@ if ! wait_for_background_jobs; then
exit 1
fi

##################
# For dynamic linking, each optimized binary needs extra exports added via
# add-export-tool. These exports (__wasm_apply_tls_relocs, __wasm_apply_global_relocs,
# __stack_pointer) are required by lind-boot to wire up the dynamically loaded
# modules at runtime.
##################
if [[ "$LIND_DYLINK" == "1" ]]; then
echo "[lmbench] adding dylink exports via add-export-tool..."
shopt -s nullglob
dylink_opts=("$LM_BENCH_BIN_DIR"/*.opt.wasm)
shopt -u nullglob
for w in "${dylink_opts[@]}"; do
"$ADD_EXPORT_TOOL" "$w" "$w" __wasm_apply_tls_relocs func __wasm_apply_tls_relocs optional
"$ADD_EXPORT_TOOL" "$w" "$w" __wasm_apply_global_relocs func __wasm_apply_global_relocs optional
"$ADD_EXPORT_TOOL" "$w" "$w" __stack_pointer global __stack_pointer
done
fi

##################
# Issue #127 requires the final staged executable names to come from `.cwasm`
# outputs. We therefore require `lind-boot --precompile`, fail the build if it
Expand Down
124 changes: 103 additions & 21 deletions nginx/compile_nginx.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ set -euo pipefail
# 2. Apply patches for WASI compatibility
# 3. Build nginx with wasm32-wasi toolchain
# 4. Optimize with wasm-opt (asyncify for multi-process support)
# 5. Precompile with wasmtime compile
# 5. Precompile with lind-boot --precompile
#
# Dynamic loading support:
# Set LIND_DYLINK=1 to build nginx as a position-independent executable
# for the dylink branch. This adds -fPIC to CFLAGS, uses PIE link flags,
# and applies the dylink-aware wasm-opt passes.
#
# Prerequisites:
# - Run 'make preflight' and 'make merge-sysroot' from lind-wasm-apps root
Expand Down Expand Up @@ -57,6 +62,7 @@ JOBS="${JOBS:-$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN || echo 4)}"
# skipping .cwasm generation while the default `ARTIFACT_MODE=full` performs
# optimization and .cwasm binary generation
##################
LIND_DYLINK="${LIND_DYLINK:-0}"
ARTIFACT_MODE="${ARTIFACT_MODE:-full}"
FORCE_CLEAN="${FORCE_CLEAN:-0}"
FORCE_CONFIGURE="${FORCE_CONFIGURE:-0}"
Expand Down Expand Up @@ -107,22 +113,71 @@ CFLAGS_WASM="-O2 -g -pthread -matomics -mbulk-memory \
-I$MERGED_SYSROOT/include/wasm32-wasi \
-D_GNU_SOURCE"

# LDFLAGS for WASM linking
# Note: nginx's generated Makefile does not use $(LDFLAGS). Instead, the link
# rule uses $(LINK) as the full linker command. We pass these flags via the
# LINK make override at build time (not via --with-ld-opt, which configure
# would test and fail for WASM-specific flags).
LDFLAGS_WASM="-Wl,--shared-memory,--import-memory,--export-memory,--max-memory=67108864 \
-Wl,--export=__stack_pointer,--export=__stack_low,--export=__tls_base \
-L$MERGED_SYSROOT/lib/wasm32-wasi \
-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
if [[ "$LIND_DYLINK" == "1" ]]; then
echo "[nginx] Dynamic linking mode enabled (LIND_DYLINK=1)"
CFLAGS_WASM+=" -fPIC"

# add-export-tool is used after wasm-opt to export relocation and stack symbols
ADD_EXPORT_TOOL="$LIND_WASM_ROOT/tools/add-export-tool/add-export-tool"
if [[ ! -x "$ADD_EXPORT_TOOL" ]]; then
echo "[nginx] ERROR: add-export-tool not found at '$ADD_EXPORT_TOOL'" >&2
exit 1
fi

# Extra CRT objects required for dynamic PIE executables
DYLINK_CRT_OBJS=(
"$MERGED_SYSROOT/lib/wasm32-wasi/set_stack_pointer.o"
"$MERGED_SYSROOT/lib/wasm32-wasi/crt1_shared.o"
"$MERGED_SYSROOT/lib/wasm32-wasi/lind_utils.o"
)
for obj in "${DYLINK_CRT_OBJS[@]}"; do
if [[ ! -f "$obj" ]]; then
echo "[nginx] ERROR: required dylink CRT object '$obj' not found." >&2
echo "[nginx] Hint: rebuild sysroot on the dylink branch (make sysroot in lind-wasm)." >&2
exit 1
fi
done

# LDFLAGS for WASM dynamic/PIE linking
# -nostartfiles: skip default CRT, use dylink-specific CRT objects instead
# -Wl,-pie: produce a Position Independent Executable
# -Wl,--import-table: import the indirect-function table (cross-module calls)
# -Wl,--allow-undefined + --unresolved-symbols=import-dynamic: resolve at load time
# -Wl,--export=__wasm_call_ctors: let lind-boot call constructors
LDFLAGS_WASM="-nostartfiles \
-Wl,-pie \
-Wl,--import-table \
-Wl,--import-memory \
-Wl,--export-memory \
-Wl,--shared-memory \
-Wl,--max-memory=67108864 \
-Wl,--allow-undefined \
-Wl,--unresolved-symbols=import-dynamic \
-Wl,--export=__wasm_call_ctors \
-Wl,--export-if-defined=__wasm_init_tls \
-Wl,--export=__tls_base \
-L$MERGED_SYSROOT/lib/wasm32-wasi \
-L$MERGED_SYSROOT/usr/lib/wasm32-wasi \
${DYLINK_CRT_OBJS[*]}"
else
# LDFLAGS for static WASM linking
# Note: nginx's generated Makefile does not use $(LDFLAGS). Instead, the link
# rule uses $(LINK) as the full linker command. We pass these flags via the
# LINK make override at build time (not via --with-ld-opt, which configure
# would test and fail for WASM-specific flags).
LDFLAGS_WASM="-Wl,--shared-memory,--import-memory,--export-memory,--max-memory=67108864 \
-Wl,--export=__stack_pointer,--export=__stack_low,--export=__tls_base \
-L$MERGED_SYSROOT/lib/wasm32-wasi \
-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
fi

echo "[nginx] using CLANG = $CLANG"
echo "[nginx] using AR = $AR"
echo "[nginx] LIND_WASM_ROOT = $LIND_WASM_ROOT"
echo "[nginx] merged sysroot = $MERGED_SYSROOT"
echo "[nginx] output dir = $NGINX_OUT_DIR"
echo "[nginx] artifact mode = $ARTIFACT_MODE"
echo "[nginx] dylink mode = $LIND_DYLINK"
echo

##################
Expand Down Expand Up @@ -213,6 +268,7 @@ clang=$CLANG
merged=$MERGED_SYSROOT
cflags=$CFLAGS_WASM
ldflags=$LDFLAGS_WASM
dylink=$LIND_DYLINK
args=$configure_args
EOF
}
Expand Down Expand Up @@ -810,15 +866,26 @@ if [[ ! -x "$WASM_OPT" ]]; then
exit 1
fi

echo "[nginx] running wasm-opt (asyncify + epoch injection) to produce runnable nginx.wasm..."
"$WASM_OPT" \
--fpcast-emu \
--epoch-injection \
--asyncify \
--debuginfo \
-O2 \
"$NGINX_RAW_WASM" \
-o "$NGINX_WASM"
if [[ "$LIND_DYLINK" == "1" ]]; then
echo "[nginx] running wasm-opt (dylink-aware: epoch-import + asyncify-import-globals)..."
"$WASM_OPT" \
--enable-bulk-memory --enable-threads \
Copy link
Copy Markdown
Contributor

@vidyalakshmir vidyalakshmir Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also noticed that there is no --fpcast-emu flag for the dynamic build.

Copy link
Copy Markdown
Contributor Author

@rishabhBudhouliya rishabhBudhouliya Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I missed that. I also didn't test the network lmbench tests also so didn't notice a failure.

--epoch-injection --pass-arg=epoch-import --pass-arg=epoch-main-module \
--asyncify --pass-arg=asyncify-import-globals \
--debuginfo \
"$NGINX_RAW_WASM" \
-o "$NGINX_WASM"
else
echo "[nginx] running wasm-opt (asyncify + epoch injection) to produce runnable nginx.wasm..."
"$WASM_OPT" \
--fpcast-emu \
--epoch-injection \
--asyncify \
--debuginfo \
-O2 \
"$NGINX_RAW_WASM" \
-o "$NGINX_WASM"
fi
echo "[nginx] optimized runnable module: $NGINX_WASM"

if [[ "$ARTIFACT_MODE" == "full" ]]; then
Expand All @@ -834,6 +901,21 @@ if [[ ! -f "$NGINX_OPT_WASM" ]]; then
exit 1
fi

###############################################################################
# 5.5. Add dylink exports via add-export-tool (dynamic mode only)
###############################################################################

if [[ "$LIND_DYLINK" == "1" ]]; then
echo "[nginx] adding dylink exports via add-export-tool..."
"$ADD_EXPORT_TOOL" "$NGINX_OPT_WASM" "$NGINX_OPT_WASM" \
__wasm_apply_tls_relocs func __wasm_apply_tls_relocs optional
"$ADD_EXPORT_TOOL" "$NGINX_OPT_WASM" "$NGINX_OPT_WASM" \
__wasm_apply_global_relocs func __wasm_apply_global_relocs optional
"$ADD_EXPORT_TOOL" "$NGINX_OPT_WASM" "$NGINX_OPT_WASM" \
__stack_pointer global __stack_pointer
echo "[nginx] dylink exports added to $NGINX_OPT_WASM"
fi

###############################################################################
# 6. cwasm generation (best-effort)
###############################################################################
Expand Down Expand Up @@ -875,7 +957,7 @@ daemon off;
master_process off;
worker_processes 1;

error_log /dev/stderr info;
error_log /var/log/nginx/error.log info;

events {
use poll;
Expand All @@ -886,7 +968,7 @@ http {
include mime.types;
default_type application/octet-stream;

access_log /dev/stdout;
access_log /var/log/nginx/access.log;

sendfile off;
keepalive_timeout 65;
Expand Down
⚡