Build and test python

Makefile

@@ -39,11 +39,11 @@ LINDFS_ROOT    := $(LIND_WASM_ROOT)/lindfs
 #   make check-build                # runs the full TESTABLE_APPS list
 #   make check-build APP=nginx      # runs a single app on demand
 #   make check-build APP="nginx grep sed"  # runs multiple apps on demand
-TESTABLE_APPS  := bash coreutils curl git grep lmbench sed tinycc
+TESTABLE_APPS  := bash coreutils curl git grep lmbench sed tinycc cpython
 APP            ?= $(TESTABLE_APPS)
 
 # -------- Phonies -------------------------------------------------------------
-.PHONY: all preflight dirs print-config check-build libtirpc gnulib zlib openssl libcxx merge-base-sysroot merge-sysroot lmbench bash nginx coreutils cpython git curl grep sed gcc binutils postgres clean clean-all rebuild-libs rebuild-sysroot test install-bash install-nginx install-git install-curl install-grep install-sed install-lmbench install-coreutils install-gcc install-binutils install
+.PHONY: all preflight dirs print-config check-build libtirpc gnulib zlib openssl libcxx merge-base-sysroot merge-sysroot lmbench bash nginx coreutils cpython git curl grep sed gcc binutils postgres clean clean-all rebuild-libs rebuild-sysroot test install-bash install-nginx install-cpython install-git install-curl install-grep install-sed install-lmbench install-coreutils install-gcc install-binutils install
 
 all: preflight libtirpc gnulib merge-sysroot lmbench bash
 
@@ -337,9 +337,12 @@ rebuild-sysroot:
 	rm -f '$(MERGE_BASE_STAMP)' '$(MERGE_TIRPC_STAMP)' '$(MERGE_GNULIB_STAMP)' '$(MERGE_ZLIB_STAMP)' '$(MERGE_OPENSSL_STAMP)' '$(MERGE_LIBCXX_STAMP)' '$(MERGE_ALL_STAMP)'
 
 # ---------------- cpython (WASM build) ----------------------------------------
-# Placeholder target to preserve the per-app staging/layering pattern.
+# Uses cpython/compile_python.sh to build python. If LIND_DYLINK=1 is set, then produces
+# a dynamically linked python binary else a statically linked python binary
 cpython: merge-sysroot
-	mkdir -p '$(APPS_BIN_DIR)/cpython/wasm32-wasi'
+	mkdir -p '$(APPS_BIN_DIR)/cpython'
+	. '$(TOOL_ENV)'
+	JOBS='$(JOBS)' '$(APPS_ROOT)/cpython/compile_python.sh'
 
 # ---------------- postgres (WASM build) ---------------------------------------
 # Placeholder target to preserve the per-app staging/layering pattern.
@@ -376,4 +379,7 @@ install-gcc:
 install-binutils:
 	'$(APPS_ROOT)/scripts/post_install.sh' '$(LINDFS_ROOT)' '$(APPS_BUILD)' binutils
 
-install: install-bash install-nginx install-git install-curl install-grep install-sed install-lmbench install-coreutils install-gcc install-binutils
+install-cpython:
+	'$(APPS_ROOT)/scripts/post_install.sh' '$(LINDFS_ROOT)' '$(APPS_BUILD)' cpython
+
+install: install-bash install-nginx install-cpython install-git install-curl install-grep install-sed install-lmbench install-coreutils install-gcc install-binutils

cpython/.gitignore

@@ -172,3 +172,7 @@ Python/frozen_modules/MANIFEST
 # People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs.
 /.claude/
 CLAUDE.local.md
+
+build-native/
+*.wasm
+*.cwasm

cpython/build_python_native.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BUILD="$SRC/build-native"
+
+mkdir -p "$BUILD"
+cd "$BUILD"
+
+"$SRC/configure" --prefix="$BUILD/install"
+make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu)
+
+echo "native python: $BUILD/python"

cpython/build_python_wasm.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+APPS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Default LIND_WASM_ROOT to parent directory (layout: lind-wasm/lind-wasm-apps)
+if [[ -z "${LIND_WASM_ROOT:-}" ]]; then
+  LIND_WASM_ROOT="$(cd "$APPS_ROOT/.." && pwd)"
+fi
+SYSROOT="${LIND_WASM_ROOT}/src/glibc/sysroot"
+LINDFS="${LIND_WASM_ROOT}/lindfs"
+LIND_BOOT="${LIND_WASM_ROOT}/src/lind-boot/target/debug/lind-boot"
+WASM_OPT="${LIND_WASM_ROOT}/tools/binaryen/bin/wasm-opt"
+
+CLANG_BIN="${LIND_WASM_ROOT}/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04/bin/clang"
+
+LIND_DYLINK="${LIND_DYLINK:-0}"
+PYTHON_OUT_DIR="$APPS_ROOT/build/cpython"
+
+cd $SCRIPT_DIR
+
+echo "Starting python build process..."
+
+# build native python if not exist
+if [ ! -f "./build-native/python" ]; then
+  echo "Building native python"
+  ./build_python_native.sh
+fi
+
+# create wasi related placeholder library
+llvm-ar crs "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-getpid.a"
+llvm-ar crs "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-signal.a"
+llvm-ar crs "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-process-clocks.a"
+
+mkdir -p build-wasm
+cd build-wasm
+
+# run configure if Makefile does not exist
+if [ ! -f "Makefile" ]; then
+  ../configure \
+    --host=wasm32-unknown-wasi \
+    --build=x86_64-unknown-linux-gnu \
+    --with-build-python=../build-native/python \
+    CC="${CLANG_BIN} \
+    -pthread \
+    --target=wasm32-unknown-wasi \
+    --sysroot ${SYSROOT} \
+    -Wl,--import-memory,--export-memory,--max-memory=67108864,--export=__stack_pointer,--export=__stack_low -D _FILE_OFFSET_BITS=64 -D __USE_LARGEFILE64 -g -O0 -fPIC" \
+    ac_cv_func_working_mktime=yes \
+    ac_cv_func_mmap_fixed_mapped=yes \
+    bash_cv_func_sigsetjmp=no \
+    ac_cv_func_localeconv=no \
+    ac_cv_func_uselocale=no \
+    ac_cv_func_setlocale=no \
+    ac_cv_func_newlocale=no \
+    ac_cv_have_chflags=no \
+    ax_cv_c_float_words_bigendian=no \
+    ac_cv_file__dev_ptmx=no \
+    ac_cv_file__dev_ptc=no \
+    ac_cv_func_memfd_create=no \
+    ac_cv_func_eventfd=no \
+    ac_cv_func_timerfd_create=no \
+    --verbose
+fi
+
+# build python
+make AR="llvm-ar" ARFLAGS="crs"
+
+# install necessary files into lind filesystem
+make install DESTDIR="${PYTHON_OUT_DIR}"
+
+
+# 6. Apply wasm-opt (best-effort)
+###############################################################################
+
+if [[ "$LIND_DYLINK" == "1" ]]; then
+    exit 0
+fi
+
+PYTHON_WASM="$SCRIPT_DIR/build-wasm/python.wasm"
+PYTHON_OPT_WASM="$SCRIPT_DIR/build-wasm/python.opt.wasm"
+PYTHON_OPT_CWASM="$SCRIPT_DIR/build-wasm/python.opt.cwasm"
+
+if [[ -x "$WASM_OPT" ]]; then
+  echo "[python] running wasm-opt (best-effort)..."
+
+  "$WASM_OPT" --epoch-injection --asyncify -O2 --debuginfo \
+    "$PYTHON_WASM" -o "$PYTHON_OPT_WASM"
+else
+  echo "[python] ERROR: wasm-opt not found; skipping optimization step and exiting."
+  exit 1
+fi
+
+if [[ ! -f "$PYTHON_OPT_WASM" ]]; then
+  echo "[python] ERROR: Failed to generate $PYTHON_OPT_WASM; Exiting.."
+  exit 1
+fi
+
+# 7. cwasm generation via lind-boot (best-effort)
+###############################################################################
+if [[ -x "$LIND_BOOT" ]]; then
+  echo "=> generating cwasm via lind-boot --precompile..."
+
+  # Pass the (potentially optimized) Wasm file to lind-boot
+  if "$LIND_BOOT" --precompile "$PYTHON_OPT_WASM"; then
+
+    if [[ -f "$PYTHON_OPT_CWASM" ]]; then
+      mkdir -p "$PYTHON_OUT_DIR/usr/local/bin"
+      cp "$PYTHON_OPT_CWASM" "$PYTHON_OUT_DIR/usr/local/bin/python"
+      echo "[python] python staged as $PYTHON_OUT_DIR/usr/local/bin/python"
+    else
+      echo "[python] ERROR: No .cwasm binary generated and no binaries copied to the build folder. Exiting .."
+      exit 1
+    fi
+
+  else
+    echo "[python] ERROR: lind-boot --precompile failed; skipping cwasm generation."
+    echo "[python] ERROR: No binaries copied to the build folder. Exiting.."
+    exit 1
+  fi
+else
+  echo "[python] NOTE: lind-boot not found at '$LIND_BOOT'; skipping cwasm generation."
+  echo "[python] ERROR: No binaries copied to the build folder. Exiting.."
+  exit 1
+fi
+
+echo
+echo "[python] build complete. Outputs under:"
+echo "  $PYTHON_OUT_DIR"
+ls -lh "$PYTHON_OUT_DIR" || true

cpython/build_shared_python_wasm.sh

@@ -0,0 +1,175 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+APPS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+if [[ -z "${LIND_WASM_ROOT:-}" ]]; then
+  LIND_WASM_ROOT="$(cd "$APPS_ROOT/.." && pwd)"
+fi
+
+LINDFS_ROOT="${LINDFS_ROOT:-$LIND_WASM_ROOT/lindfs}"
+
+BUILD_WASM="$SCRIPT_DIR/build-wasm"
+SYSROOT="$LIND_WASM_ROOT/src/glibc/sysroot"
+LIND_BOOT="${LIND_WASM_ROOT}/src/lind-boot/target/debug/lind-boot"
+CLANG_BIN="${LIND_WASM_ROOT}/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04/bin/clang"
+WASM_OPT="${WASM_OPT:-$LIND_WASM_ROOT/tools/binaryen/bin/wasm-opt}"
+ADD_EXPORT_TOOL="$LIND_WASM_ROOT/tools/add-export-tool/add-export-tool"
+
+PYTHON_OUT_DIR="$APPS_ROOT/build/cpython"
+STATIC_LIB="$BUILD_WASM/libpython3.14.a"
+DYNAMIC_LIB_WASM="$BUILD_WASM/libpython3.14.wasm"
+DYNAMIC_LIB_OPT="$BUILD_WASM/libpython3.14.opt.wasm"
+DYNAMIC_LIB_OPT_CWASM="$BUILD_WASM/libpython3.14.opt.cwasm"
+DYNAMIC_STAGED_LIB="$PYTHON_OUT_DIR/lib/libpython3.14.so"
+mkdir -p "$PYTHON_OUT_DIR/lib"
+mkdir -p "$PYTHON_OUT_DIR/usr/local/bin"
+# compile shared libpython
+$CLANG_BIN \
+    --target=wasm32-unknown-wasi \
+    -fPIC \
+    --sysroot "$SYSROOT" \
+    -fvisibility=default \
+    -Wl,--import-memory \
+    -Wl,--shared-memory \
+    -Wl,--export-dynamic \
+    -Wl,--experimental-pic \
+    -Wl,--unresolved-symbols=import-dynamic \
+    -Wl,-shared \
+    -Wl,--whole-archive \
+    "$STATIC_LIB" \
+    -Wl,--no-whole-archive \
+    "$LIND_WASM_ROOT/src/glibc/build/lind_debug.o" \
+    -g -O0 -o "$DYNAMIC_LIB_WASM" 	|| { echo "[python] ERROR: clang compilation failed"; exit 1; }
+
+
+if [[ ! -f "$DYNAMIC_LIB_WASM" ]]; then
+  echo "[python] ERROR: Failed to generate '$DYNAMIC_LIB_WASM'; Exiting.."
+  exit 1
+fi
+
+"$ADD_EXPORT_TOOL" "$DYNAMIC_LIB_WASM" "$DYNAMIC_LIB_WASM" __wasm_apply_tls_relocs func __wasm_apply_tls_relocs optional || { echo "[python] ERROR: add-export-tool tls failed"; exit 1; }
+
+"$ADD_EXPORT_TOOL" "$DYNAMIC_LIB_WASM" "$DYNAMIC_LIB_WASM" __wasm_apply_global_relocs func __wasm_apply_global_relocs optional || { echo "[python] ERROR: add-export-tool global failed"; exit 1; }
+
+"$ADD_EXPORT_TOOL" "$DYNAMIC_LIB_WASM" "$DYNAMIC_LIB_WASM"  __stack_pointer global __stack_pointer optional || { echo "[python] ERROR: add-export-tool stack pointer failed"; exit 1; }
+
+
+$WASM_OPT --enable-bulk-memory --enable-threads --epoch-injection --pass-arg=epoch-import --asyncify --pass-arg=asyncify-import-globals -O2 --debuginfo "$DYNAMIC_LIB_WASM" -o "$DYNAMIC_LIB_OPT" || { echo "[python] ERROR: wasm-opt failed on '$DYNAMIC_LIB_OPT'; Exiting.."; exit 1; }
+
+if [[ ! -f "$DYNAMIC_LIB_OPT" ]]; then
+  echo "[python] ERROR: Failed to generate '$DYNAMIC_LIB_OPT'; Exiting.."
+  exit 1
+fi
+
+# do precompile
+$LIND_WASM_ROOT/scripts/lind_compile --precompile-only "$DYNAMIC_LIB_OPT"|| { echo "[python] ERROR: lind_compile failed on '$DYNAMIC_LIB_OPT_CWASM'; Exiting.."; exit 1; }
+
+if [[ ! -f "$DYNAMIC_LIB_OPT_CWASM" ]]; then
+  echo "[python] ERROR: Failed to generate '$DYNAMIC_LIB_OPT_CWASM'; Exiting.."
+  exit 1
+fi
+
+cp "$DYNAMIC_LIB_OPT_CWASM" "$DYNAMIC_STAGED_LIB"
+echo "[python] Dynamic shared library staged as $DYNAMIC_STAGED_LIB"
+
+
+
+#compile shared python
+
+PYTHON_WASM="$BUILD_WASM/python_shared.wasm"
+PYTHON_OPT_WASM="$BUILD_WASM/python_shared.opt.wasm"
+PYTHON_OPT_CWASM="$BUILD_WASM/python_shared.opt.cwasm"
+
+$CLANG_BIN \
+    -pthread \
+    -fPIC \
+    --target=wasm32-unknown-wasi \
+    --sysroot "$SYSROOT" \
+    -Wl,-pie \
+    -Wl,--import-table \
+    -Wl,--import-memory \
+    -Wl,--export-memory \
+    -Wl,--shared-memory \
+    -Wl,--max-memory=67108864 \
+    -Wl,--export=__stack_pointer \
+    -Wl,--export=__stack_low \
+    -Wl,--export=__wasm_call_ctors \
+    -Wl,--export-if-defined=__wasm_init_tls \
+    -Wl,--export=__tls_base \
+    -Wl,--allow-undefined \
+    -Wl,--unresolved-symbols=import-dynamic \
+    -D _FILE_OFFSET_BITS=64 \
+    -D __USE_LARGEFILE64 \
+    -g -O0 \
+    -o "$BUILD_WASM/python_shared.wasm" \
+    "$BUILD_WASM/Programs/python.o" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_Hash_SHA2.a" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_Hash_SHA1.a" \
+    "$BUILD_WASM/Modules/expat/libexpat.a" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_Hash_MD5.a" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_Hash_BLAKE2.a" \
+    "$BUILD_WASM/Modules/_decimal/libmpdec/libmpdec.a" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_HMAC.a" \
+    "$BUILD_WASM/Modules/_hacl/libHacl_Hash_SHA3.a" \
+    -ldl -lwasi-emulated-signal -lwasi-emulated-getpid -lwasi-emulated-process-clocks -lpthread -lm
+
+
+if [[ -x "$WASM_OPT" ]]; then
+  echo "[python] running wasm-opt (best-effort)..."
+
+  "$WASM_OPT" \
+      --enable-bulk-memory --enable-threads \
+      --epoch-injection --pass-arg=epoch-import --pass-arg=epoch-main-module \
+      --asyncify --pass-arg=asyncify-import-globals \
+      --debuginfo \
+      "$PYTHON_WASM" -o "$PYTHON_OPT_WASM" || true
+else
+  echo "[python] ERROR: wasm-opt not found; skipping optimization step and exiting."
+  exit 1
+fi
+
+if [[ ! -f "$PYTHON_OPT_WASM" ]]; then
+  echo "[python] ERROR: Failed to generate $PYTHON_OPT_WASM; Exiting.."
+  exit 1
+fi
+
+echo "[python] adding dylink exports via add-export-tool..."
+"$ADD_EXPORT_TOOL" "$PYTHON_OPT_WASM" "$PYTHON_OPT_WASM" __wasm_apply_tls_relocs func __wasm_apply_tls_relocs optional
+"$ADD_EXPORT_TOOL" "$PYTHON_OPT_WASM" "$PYTHON_OPT_WASM" __wasm_apply_global_relocs func __wasm_apply_global_relocs optional
+"$ADD_EXPORT_TOOL" "$PYTHON_OPT_WASM" "$PYTHON_OPT_WASM" __stack_pointer global __stack_pointer
+
+# 7. cwasm generation via lind-boot (best-effort)
+###############################################################################
+if [[ -x "$LIND_BOOT" ]]; then
+  echo "=> generating cwasm via lind-boot --precompile..."
+
+  # Pass the (potentially optimized) Wasm file to lind-boot
+  if "$LIND_BOOT" --precompile "$PYTHON_OPT_WASM"; then
+
+    if [[ -f "$PYTHON_OPT_CWASM" ]]; then
+      cp "$PYTHON_OPT_CWASM" "$PYTHON_OUT_DIR/usr/local/bin/python"
+      echo "[python] python staged as $PYTHON_OUT_DIR/usr/local/bin/python"
+    else
+      echo "[python] ERROR: No .cwasm binary generated and no binaries copied to the build folder. Exiting .."
+      exit 1
+    fi
+
+  else
+    echo "[python] ERROR: lind-boot --precompile failed; skipping cwasm generation."
+    echo "[python] ERROR: No binaries copied to the build folder. Exiting.."
+    exit 1
+  fi
+else
+  echo "[python] NOTE: lind-boot not found at '$LIND_BOOT'; skipping cwasm generation."
+  echo "[python] ERROR: No binaries copied to the build folder. Exiting.."
+  exit 1
+fi
+
+echo
+echo "[python] build complete. Outputs under:"
+echo "  $PYTHON_OUT_DIR"
+ls -lh "$PYTHON_OUT_DIR" || true
+
+

cpython/clean.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+APPS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+STAGE_DIR="$APPS_ROOT/build/cpython"
+
+# Clean native and wasm build
+rm -rf $SCRIPT_DIR/build-native
+rm -rf $SCRIPT_DIR/build-wasm
+
+# 3. Clean WASI placeholder libraries
+rm -f "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-getpid.a"
+rm -f "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-signal.a"
+rm -f "${SYSROOT}/lib/wasm32-wasi/libwasi-emulated-process-clocks.a"
+
+
+# Clean the Lind filesystem installation
+sudo rm -rf "$STAGE_DIR"
+

cpython/compile_python.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+$SCRIPT_DIR/build_python_native.sh
+$SCRIPT_DIR/build_python_wasm.sh
+if [[ "$LIND_DYLINK" == "1" ]]; then
+	$SCRIPT_DIR/build_shared_python_wasm.sh
+fi

cpython/run.sh

@@ -0,0 +1 @@
+/home/lind/lind-wasm/src/wasmtime/target/debug/wasmtime run --wasi threads=y --wasi preview2=n --preload env=/lib/libc.so --preload env=/lib/libm.so --preload env=/lib/libpython3.14.so python.wasm pytests/add.py