feat(horizonxj3): Support horizon x3 pi (#1597)

* feat(horizonXJ3): export onnx and convert to bin

* feat(horizonxj3): add quantization pipeline

* fest(horizonxj3): minor update

* feat(horizonxj3): add runtime, 1st commit

* fest(horizonxj3): rename horizonxj3 -> horizonbpu

* fest(horizonxj3): pipeline stage && nchw->nhwc

* fest(horizonxj3): mhwc -> nchw

* fest(horizonxj3): remove attn_run_on_bpu

* fest(horizonxj3): rm grpc

* fest(horizonxj3): fix att_mask shape

* fest(horizonxj3): fix runtime

* fest(horizonxj3): remove XxxManager from being class members

* fest(horizonxj3): rename X3??? to BPU???

* fest(horizonxj3): add args: extra_ops_run_on_cpu

* fest(horizonxj3): refine readme

* fest(horizonxj3): refine readme

* fest(horizonxj3): refine readme

* fix(run_on_cpu): [cr_id_skip] fix run_on_cpu for torch < 1.12.0

* refactor(horizonxj3): update readme

* refactor(horizonxj3): update doc

* refactor(horizonxj3): update readme

* fix(run_on_cpu): [cr_id_skip] fix run_on_cpu for torch < 1.13.0

* fix(r3pi): Fix readme

* refactor(horizonxj3): Move core/decoder/bpu_asr_model.* to horizonbpu/bpu/bpu_asr_model.*

* fix(horizonxj3): Fix lint

Author: xingchensong

runtime/core/cmake/bpu.cmake

@@ -0,0 +1,30 @@
+if(BPU)
+  if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+      set(EASY_DNN_URL "https://github.com/xingchensong/toolchain_pkg/releases/download/easy_dnn/easy_dnn.0.4.11.tar.gz")
+      set(URL_HASH "SHA256=a1a6f77d1baae7181d75ec5d37a2ee529ac4e1c4400babd6ceb1c007392a4904")
+    else()
+      message(FATAL_ERROR "Unsupported CMake System Processor '${CMAKE_SYSTEM_PROCESSOR}' (expected 'aarch64')")
+    endif()
+  else()
+    message(FATAL_ERROR "Unsupported CMake System Name '${CMAKE_SYSTEM_NAME}' (expected 'Linux')")
+  endif()
+
+  FetchContent_Declare(easy_dnn
+    URL ${EASY_DNN_URL}
+    URL_HASH ${URL_HASH}
+  )
+  FetchContent_MakeAvailable(easy_dnn)
+  include_directories(${easy_dnn_SOURCE_DIR}/easy_dnn/0.4.11_linux_aarch64-j3_hobot_gcc6.5.0/files/easy_dnn/include)
+  include_directories(${easy_dnn_SOURCE_DIR}/dnn/1.7.0_linux_aarch64-j3_hobot_gcc6.5.0/files/dnn/include)
+  include_directories(${easy_dnn_SOURCE_DIR}/hlog/0.4.7_linux_aarch64-j3_hobot_gcc6.5.0/files/hlog/include)
+  link_directories(${easy_dnn_SOURCE_DIR}/easy_dnn/0.4.11_linux_aarch64-j3_hobot_gcc6.5.0/files/easy_dnn/lib)
+  link_directories(${easy_dnn_SOURCE_DIR}/dnn/1.7.0_linux_aarch64-j3_hobot_gcc6.5.0/files/dnn/lib)
+  link_directories(${easy_dnn_SOURCE_DIR}/hlog/0.4.7_linux_aarch64-j3_hobot_gcc6.5.0/files/hlog/lib)
+
+  add_definitions(-DUSE_BPU)
+  # NOTE(xcsong): Reasons for adding flag `-fuse-ld=gold`:
+  #   https://stackoverflow.com/questions/59915966/unknown-gcc-linker-error-but-builds-sucessfully/59916438#59916438
+  #   https://github.com/tensorflow/tensorflow/issues/47849
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fuse-ld=gold")
+endif()

runtime/core/decoder/CMakeLists.txt

@@ -7,8 +7,8 @@ set(decoder_srcs
   ctc_endpoint.cc
 )
 
-if(NOT TORCH AND NOT ONNX AND NOT XPU AND NOT IOS)
-  message(FATAL_ERROR "Please build with TORCH or ONNX or XPU or IOS!!!")
+if(NOT TORCH AND NOT ONNX AND NOT XPU AND NOT IOS AND NOT BPU)
+  message(FATAL_ERROR "Please build with TORCH or ONNX or XPU or IOS or BPU!!!")
 endif()
 if(TORCH OR IOS)
   list(APPEND decoder_srcs torch_asr_model.cc)
@@ -30,6 +30,9 @@ else()
   if(ONNX)
     target_link_libraries(decoder PUBLIC onnxruntime)
   endif()
+  if(BPU)
+    target_link_libraries(decoder PUBLIC bpu_asr_model)
+  endif()
   if(XPU)
     target_link_libraries(decoder PUBLIC xpu_conformer)
   endif()

runtime/core/decoder/params.h

@@ -31,6 +31,9 @@
 #ifdef USE_XPU
 #include "xpu/xpu_asr_model.h"
 #endif
+#ifdef USE_BPU
+#include "bpu/bpu_asr_model.h"
+#endif
 #include "frontend/feature_pipeline.h"
 #include "post_processor/post_processor.h"
 #include "utils/flags.h"
@@ -45,6 +48,9 @@ DEFINE_string(onnx_dir, "", "directory where the onnx model is saved");
 // XPUAsrModel flags
 DEFINE_string(xpu_model_dir, "",
               "directory where the XPU model and weights is saved");
+// BPUAsrModel flags
+DEFINE_string(bpu_model_dir, "",
+              "directory where the HORIZON BPU model is saved");
 
 // FeaturePipelineConfig flags
 DEFINE_int32(num_bins, 80, "num mel bins for fbank feature");
@@ -156,9 +162,18 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
     resource->model = model;
 #else
     LOG(FATAL) << "Please rebuild with cmake options '-DXPU=ON'.";
+#endif
+  } else if (!FLAGS_bpu_model_dir.empty()) {
+#ifdef USE_BPU
+    LOG(INFO) << "Reading Horizon BPU model from " << FLAGS_bpu_model_dir;
+    auto model = std::make_shared<BPUAsrModel>();
+    model->Read(FLAGS_bpu_model_dir);
+    resource->model = model;
+#else
+    LOG(FATAL) << "Please rebuild with cmake options '-DBPU=ON'.";
 #endif
   } else {
-    LOG(FATAL) << "Please set ONNX, TORCH or XPU model path!!!";
+    LOG(FATAL) << "Please set ONNX, TORCH, XPU or BPU model path!!!";
   }
 
   LOG(INFO) << "Reading unit table " << FLAGS_unit_path;

runtime/horizonbpu/.gitignore

@@ -0,0 +1,2 @@
+build/
+fc_base/

runtime/horizonbpu/CMakeLists.txt

@@ -0,0 +1,67 @@
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+project(wenet VERSION 0.1)
+
+option(CXX11_ABI "whether to use CXX11_ABI libtorch" OFF)
+option(GRAPH_TOOLS "whether to build TLG graph tools" OFF)
+option(BUILD_TESTING "whether to build unit test" ON)
+
+option(GRPC "whether to build with gRPC" OFF)
+# TODO(Binbin Zhang): Change websocket to OFF since it depends on boost
+#                     which is a very big library
+option(WEBSOCKET "whether to build with websocket" OFF)
+option(BPU "whether to build with BPU" ON)
+
+set(CMAKE_VERBOSE_MAKEFILE OFF)
+
+include(FetchContent)
+set(FETCHCONTENT_QUIET OFF)
+get_filename_component(fc_base "fc_base" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(FETCHCONTENT_BASE_DIR ${fc_base})
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+# Keep the same with openfst, -fPIC or -fpic
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -pthread -fPIC")
+
+# Include all dependency
+include(openfst)
+# This CMakeLists.txt is only used for horizon bpu, so remove the contents
+#  about onnx, libtorch, gpu and windows.
+include(bpu)
+# Compile bpu_asr_model
+add_subdirectory(bpu)
+
+include_directories(
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/kaldi
+)
+
+# Build all libraries
+add_subdirectory(utils)
+add_subdirectory(frontend)
+add_subdirectory(post_processor)
+add_subdirectory(kaldi)  # kaldi: wfst based decoder
+add_subdirectory(decoder)
+add_subdirectory(api)
+
+# Optionally, you can build with websocket
+if(WEBSOCKET)
+  include(boost)
+  add_subdirectory(websocket)
+endif()
+
+# Optionally, you can build with gRPC
+if(GRPC)
+  include(grpc)
+  add_subdirectory(grpc)
+endif()
+
+# Build all bins
+add_subdirectory(bin)
+
+# Unit Test
+if(BUILD_TESTING)
+  include(gtest)
+  add_subdirectory(test)
+endif()

runtime/horizonbpu/README.md

@@ -0,0 +1,91 @@
+# WeNet & Horizon BPU (Cross Compile)
+
+* Step 1. Setup environment (install horizon packages and cross compile tools) in the PC. (~10min)
+
+```sh
+# Conda env (This conda env is only used for converting bpu models, not for training torch models,
+#   It's OK to install cpu-version pytorch)
+conda create -n horizonbpu python=3.8
+conda activate horizonbpu
+git clone https://github.com/wenet-e2e/wenet.git
+cd wenet/runtime/horizonbpu
+pip install -r ../../requirements.txt -i https://mirrors.aliyun.com/pypi/simple
+pip install torch==1.13.0 torchaudio==0.13.0 torchvision==0.14.0 onnx onnxruntime -i https://mirrors.aliyun.com/pypi/simple
+
+# Horizon packages
+wget https://gitee.com/xcsong-thu/toolchain_pkg/releases/download/resource/wheels.tar.gz
+tar -xzf wheels.tar.gz
+pip install wheels/* -i https://mirrors.aliyun.com/pypi/simple
+
+# Cross compile tools
+sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+```
+
+
+* Step 2. Build decoder_main. It requires cmake 3.14 or above. and Send the binary/libraries to Horizon X3PI. (~20min)
+
+``` sh
+# Assume current dir is `wenet/runtime/horizonbpu`
+cmake -B build -DBPU=ON -DONNX=OFF -DTORCH=OFF -DWEBSOCKET=OFF -DGRPC=OFF -DCMAKE_TOOLCHAIN_FILE=toolchains/aarch64-linux-gnu.toolchain.cmake
+cmake --build build
+
+# Send binary and libraries
+export BPUIP=xxx.xxx.xxx.xxx
+export DEMO_PATH_ON_BOARD=/path/to/demo
+scp build/bin/decoder_main sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+scp fc_base/easy_dnn-src/dnn/*j3*/*/*/lib/libdnn.so sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+scp fc_base/easy_dnn-src/easy_dnn/*j3*/*/*/lib/libeasy_dnn.so sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+scp fc_base/easy_dnn-src/hlog/*j3*/*/*/lib/libhlog.so sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+```
+
+* Step 3. Export model to ONNX and convert ONNX to Horizon .bin and Send the model/dict/test_wav to Horizon X3PI. (~40min)
+
+``` sh
+# Assume current dir is `wenet/runtime/horizonbpu`
+conda activate horizonbpu
+export WENET_DIR=$PWD/../../
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=$WENET_DIR:$PYTHONPATH
+export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python'
+
+# Download torch model
+wget https://ghproxy.com/https://github.com/xingchensong/toolchain_pkg/releases/download/conformer_subsample8_110M/model_subsample8_parameter110M.tar.gz
+tar -xzf model_subsample8_parameter110M.tar.gz
+
+# Convert torch model to bpu model (*.pt -> *.onnx -> *.bin)
+# NOTE(xcsong): Convert model with 110M parameters requires CPU MEM >= 16G,
+#   if your CPU does not meet the requirement, you can download pre-converted encoder.bin/ctc.bin
+#   via this link: https://github.com/xingchensong/toolchain_pkg/releases
+python3 $WENET_DIR/tools/onnx2horizonbin.py \
+  --config ./model_subsample8_parameter110M/train.yaml \
+  --checkpoint ./model_subsample8_parameter110M/final.pt \
+  --output_dir ./model_subsample8_parameter110M/sample50_chunk8_leftchunk16 \
+  --chunk_size 8 \
+  --num_decoding_left_chunks 16 \
+  --max_samples 50 \
+  --dict ./model_subsample8_parameter110M/units.txt \
+  --cali_datalist ./model_subsample8_parameter110M/calibration_data/data.list
+
+# scp test wav file and dictionary
+scp ./model_subsample8_parameter110M/test_wav.wav sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+scp ./model_subsample8_parameter110M/units.txt sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+# scp bpu models
+scp ./model_subsample8_parameter110M/sample50_chunk8_leftchunk16/hb_makertbin_output_encoder/encoder.bin sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+scp ./model_subsample8_parameter110M/sample50_chunk8_leftchunk16/hb_makertbin_output_ctc/ctc.bin sunrise@$BPUIP:$DEMO_PATH_ON_BOARD
+```
+
+* Step 4. Testing on X3PI, the RTF(real time factor) is shown in Horizon X3PI's console. (~1min)
+
+``` sh
+cd /path/to/demo
+export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH
+export GLOG_logtostderr=1
+export GLOG_v=2
+./decoder_main \
+    --chunk_size 8 \
+    --num_left_chunks 16 \
+    --rescoring_weight 0.0 \
+    --wav_path ./test_wav.wav \
+    --bpu_model_dir ./ \
+    --unit_path ./units.txt 2>&1 | tee log.txt
+```

runtime/horizonbpu/api

@@ -0,0 +1 @@
+../core/api

runtime/horizonbpu/bin

@@ -0,0 +1 @@
+../core/bin

runtime/horizonbpu/bpu/CMakeLists.txt

@@ -0,0 +1,11 @@
+message("cmake build type is ${CMAKE_BUILD_TYPE} .")
+
+if(BPU)
+  list(APPEND bpu_asr_model_srcs ./bpu_asr_model.cc)
+  message(STATUS "Use src_files: [ ${bpu_asr_model_srcs} ] to compile bpu_asr_model .")
+
+  # compile bpu_asr_model
+  include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
+  add_library(bpu_asr_model STATIC ${bpu_asr_model_srcs})
+  target_link_libraries(bpu_asr_model PUBLIC easy_dnn dnn)
+endif()