feat(SIM): Speed up sim using hier and PGO

src/test/csrc/Makefile

@@ -1,14 +1,26 @@
+# Configurations
+VERILATOR_JOBS ?= 6
+QUIET ?= 1
+EMU_THREADS ?= 16
+# Hierarchical Verilation significantly reduces verilation time while slightly decrease simulation speed.
+HIER ?= 1
+WAVE ?= 0
+# PGO, recommended. Runs ~2.5x faster
+BOLT ?= 1
+
+
+.DEFAULT_GOAL:= sim
 ROOT := $(abspath ../../..)
 RTL_DIR := $(ROOT)/build/rtl
 C_SRC_DIR := $(abspath .)
 XS_DIFFTEST_DIR := $(ROOT)/XiangShan/difftest
 XS_COMMON_DIR := $(XS_DIFFTEST_DIR)/src/test/csrc/common
 XS_CONFIG_DIR := $(XS_DIFFTEST_DIR)/config
-
+SIM = $(BUILD_DIR)/VSimTop
 BUILD_DIR ?= $(C_SRC_DIR)/obj_dir
 VERILATOR ?= verilator
-EMU_THREADS ?= 1
 
+include $(C_SRC_DIR)/script/bolt.mk
 CXX_SRCS := $(C_SRC_DIR)/sim_main.cpp \
             $(XS_COMMON_DIR)/common.cpp \
             $(XS_COMMON_DIR)/compress.cpp \
@@ -17,23 +29,43 @@ CXX_SRCS := $(C_SRC_DIR)/sim_main.cpp \
             $(XS_COMMON_DIR)/flash.cpp \
             $(XS_CONFIG_DIR)/config.cpp
 
-CFLAGS := -std=c++17 -mcmodel=medium -DNO_GZ_COMPRESSION -DNO_ZSTD_COMPRESSION \
+CFLAGS := -std=c++17 -DNO_GZ_COMPRESSION -DNO_ZSTD_COMPRESSION \
           -I$(XS_COMMON_DIR) -I$(XS_CONFIG_DIR) -I$(C_SRC_DIR)
 LDFLAGS := -pthread
+ifeq ($(BOLT),1)
+  LDFLAGS += $(PGO_LDFLAGS)
+endif
 
 VERILATOR_FLAGS := -O3 --sv --cc --exe --top-module SimTop --threads $(EMU_THREADS) \
   --Mdir $(BUILD_DIR) -I$(C_SRC_DIR) -I$(XS_COMMON_DIR) -I$(XS_CONFIG_DIR) \
-  -y $(RTL_DIR) -f $(RTL_DIR)/filelist.f -v $(C_SRC_DIR)/DifftestMacros.v --trace-fst \
+  -y $(RTL_DIR) -F $(RTL_DIR)/filelist.f -v $(C_SRC_DIR)/DifftestMacros.v \
   -CFLAGS "$(CFLAGS)" -LDFLAGS "$(LDFLAGS)" \
-  -Wno-fatal -Wno-UNOPTFLAT -Wno-BLKANDNBLK -Wno-STMTDLY -Wno-WIDTH --max-num-width 150000 --assert --x-assign unique --output-split 30000 --output-split-cfuncs 30000
+  --verilate-jobs $(VERILATOR_JOBS) \
+  -Wno-fatal -Wno-UNOPTFLAT -Wno-BLKANDNBLK -Wno-STMTDLY -Wno-WIDTH \
+  --max-num-width 150000 --assert --x-assign unique --output-split 30000 --output-split-cfuncs 30000
+VERILATOR_FLAGS += $(if $(filter 1,$(HIER)),--hierarchical $(C_SRC_DIR)/corvus_hier.vlt)
+VERILATOR_FLAGS += $(if $(filter 1,$(WAVE)),--trace-fst)
 
-SIM := $(BUILD_DIR)/VSimTop
 
 .PHONY: sim clean
-
 sim:
 	time $(VERILATOR) $(VERILATOR_FLAGS) $(CXX_SRCS)
-	time $(MAKE) VM_PARALLEL_BUILDS=1 -C $(BUILD_DIR) -f $(BUILD_DIR)/VSimTop.mk
+ifeq ($(BOLT),1)
+	@$(RM) $(SIM)
+endif
+	time $(MAKE) -C $(BUILD_DIR) -f $(BUILD_DIR)/VSimTop.mk $(if $(filter 1,$(QUIET)),OBJCACHE="@echo + CXX $$< && ccache")
+ifeq ($(BOLT),1)
+	@echo "BOLT_PGO enabled - saving original binary and applying BOLT optimization..."
+	@if [ -f "$(SIM)" ]; then \
+		mv $(SIM) $(SIM).orig; \
+		$(MAKE) -f $(C_SRC_DIR)/Makefile bolt-pgo; \
+	else \
+		echo "Error: Simulator binary not found at $(SIM)"; \
+		exit 1; \
+	fi
+endif
+	@echo SIM:$(SIM)
 
 clean:
 	$(RM) -r $(BUILD_DIR)
+	@$(MAKE) -C $(C_SRC_DIR)/app/ clean

src/test/csrc/app/.gitignore

@@ -0,0 +1,3 @@
+*.elf
+*.bin
+*.dump

src/test/csrc/app/Makefile

@@ -0,0 +1,28 @@
+TOOLCHAIN ?= riscv64-linux-gnu-
+CC := $(TOOLCHAIN)gcc
+OBJCOPY := $(TOOLCHAIN)objcopy
+OBJDUMP := $(TOOLCHAIN)objdump
+
+APP ?= uart_min
+BIN := $(APP).bin
+ELF := $(APP).elf
+ASM := $(APP).S
+LDS := link.ld
+
+CFLAGS := -march=rv64i_zicsr -mabi=lp64 -nostdlib -nostartfiles -static -ffreestanding -Wl,--no-gc-sections -Wl,--build-id=none
+
+.PHONY: all disasm clean
+
+all: $(BIN)
+
+$(ELF): $(ASM) $(LDS)
+	$(CC) $(CFLAGS) -T $(LDS) -o $@ $(ASM)
+
+$(BIN): $(ELF)
+	$(OBJCOPY) -S --set-section-flags .bss=alloc,contents -O binary $< $@
+
+disasm: $(ELF)
+	$(OBJDUMP) -D $< > $(APP).dump
+
+clean:
+	$(RM) $(ELF) $(BIN) $(APP).dump

src/test/csrc/app/link.ld

@@ -0,0 +1,24 @@
+ENTRY(_start)
+
+SECTIONS
+{
+  . = 0x80000000;
+
+  .text : {
+    *(.text*)
+    *(.rodata*)
+  }
+
+  .data : {
+    *(.data*)
+  }
+
+  .bss : {
+    *(.bss*)
+  }
+  /DISCARD/ : {
+    *(.debug*)
+    *(.comment)
+    *(.riscv.attributes)
+  }
+}

src/test/csrc/app/uart_min.S

@@ -0,0 +1,113 @@
+.section .text
+.globl _start
+
+.equ UART_BASE, 0x310b0000
+.equ UART_THR,  0
+.equ UART_IER,  4
+.equ UART_FCR,  8
+.equ UART_LCR,  12
+.equ UART_LSR,  20
+
+_start:
+  li s0, UART_BASE
+  jal ra, uart_init
+
+  li a0, '['
+  jal ra, uart_putc
+  li a0, 'h'
+  jal ra, uart_putc
+  li a0, 'a'
+  jal ra, uart_putc
+  li a0, 'r'
+  jal ra, uart_putc
+  li a0, 't'
+  jal ra, uart_putc
+
+  li a0, '0'
+  jal ra, uart_putc
+  li a0, 'x'
+  jal ra, uart_putc
+
+  csrr t0, mhartid
+  mv a0, t0
+  jal ra, uart_put_hex64
+
+  li a0, ']'
+  jal ra, uart_putc
+  li a0, ' '
+  jal ra, uart_putc
+  li a0, 'u'
+  jal ra, uart_putc
+  li a0, 'a'
+  jal ra, uart_putc
+  li a0, 'r'
+  jal ra, uart_putc
+  li a0, 't'
+  jal ra, uart_putc
+  li a0, ' '
+  jal ra, uart_putc
+  li a0, 'o'
+  jal ra, uart_putc
+  li a0, 'k'
+  jal ra, uart_putc
+  li a0, '\n'
+  jal ra, uart_putc
+
+loop:
+  j loop
+
+uart_init:
+  # Enable DLAB to program baud divisor.
+  li t0, 0x80
+  sb t0, UART_LCR(s0)
+
+  # 50MHz / (16 * 115200) ~= 27 (0x1b)
+  li t0, 0x1b
+  sb t0, UART_THR(s0)   # DLL (DLAB=1)
+  sb zero, UART_IER(s0) # DLM (DLAB=1)
+
+  # 8N1, disable UART interrupts, and enable/clear FIFO.
+  li t0, 0x03
+  sb t0, UART_LCR(s0)
+  sb zero, UART_IER(s0)
+  li t0, 0x07
+  sb t0, UART_FCR(s0)
+  ret
+
+uart_putc:
+4:
+  lbu t0, UART_LSR(s0)
+  andi t0, t0, 0x20
+  beqz t0, 4b
+  sb a0, UART_THR(s0)
+  ret
+
+uart_put_hex64:
+  mv t5, ra
+  mv t4, a0
+  li t2, 60
+  li t3, 0
+5:
+  srl t0, t4, t2
+  andi t0, t0, 0xf
+  bnez t3, 8f
+  beqz t0, 9f
+  li t3, 1
+8:
+  li t1, 10
+  bltu t0, t1, 6f
+  addi a0, t0, 87
+  j 7f
+6:
+  addi a0, t0, 48
+7:
+  jal ra, uart_putc
+9:
+  addi t2, t2, -4
+  bgez t2, 5b
+  bnez t3, 10f
+  li a0, '0'
+  jal ra, uart_putc
+10:
+  mv ra, t5
+  ret

src/test/csrc/corvus_hier.vlt

@@ -0,0 +1,5 @@
+`verilator_config
+
+// Treat XiangShan wrapper as a hierarchical block so Corvus-side iterations
+// can avoid repeatedly flattening the whole CPU internals.
+hier_block -module "XSTopWrap"