diff --git a/runtime/core/cmake/openfst.cmake b/runtime/core/cmake/openfst.cmake index 20d91116d4..3c58cddc12 100644 --- a/runtime/core/cmake/openfst.cmake +++ b/runtime/core/cmake/openfst.cmake @@ -5,53 +5,25 @@ if(NOT ANDROID) set(WITH_GFLAGS OFF CACHE BOOL "whether build glog with gflags" FORCE) include(glog) - set(CONFIG_FLAGS "") - if(NOT FST_HAVE_BIN) - if(MSVC) - set(HAVE_BIN OFF CACHE BOOL "Build the fst binaries" FORCE) - else() - set(CONFIG_FLAGS "--disable-bin") - endif() - endif() - - if(MSVC) - set(HAVE_SCRIPT OFF CACHE BOOL "Build the fstscript" FORCE) - set(HAVE_COMPACT OFF CACHE BOOL "Build compact" FORCE) - set(HAVE_CONST OFF CACHE BOOL "Build const" FORCE) - set(HAVE_GRM OFF CACHE BOOL "Build grm" FORCE) - set(HAVE_PDT OFF CACHE BOOL "Build pdt" FORCE) - set(HAVE_MPDT OFF CACHE BOOL "Build mpdt" FORCE) - set(HAVE_LINEAR OFF CACHE BOOL "Build linear" FORCE) - set(HAVE_LOOKAHEAD OFF CACHE BOOL "Build lookahead" FORCE) - set(HAVE_NGRAM OFF CACHE BOOL "Build ngram" FORCE) - set(HAVE_SPECIAL OFF CACHE BOOL "Build special" FORCE) + if(NOT GRAPH_TOOLS) + set(HAVE_BIN OFF CACHE BOOL "Build the fst binaries" FORCE) endif() + set(HAVE_SCRIPT OFF CACHE BOOL "Build the fstscript" FORCE) + set(HAVE_COMPACT OFF CACHE BOOL "Build compact" FORCE) + set(HAVE_CONST OFF CACHE BOOL "Build const" FORCE) + set(HAVE_GRM OFF CACHE BOOL "Build grm" FORCE) + set(HAVE_PDT OFF CACHE BOOL "Build pdt" FORCE) + set(HAVE_MPDT OFF CACHE BOOL "Build mpdt" FORCE) + set(HAVE_LINEAR OFF CACHE BOOL "Build linear" FORCE) + set(HAVE_LOOKAHEAD OFF CACHE BOOL "Build lookahead" FORCE) + set(HAVE_NGRAM OFF CACHE BOOL "Build ngram" FORCE) + set(HAVE_SPECIAL OFF CACHE BOOL "Build special" FORCE) # The original openfst uses GNU Build System to run configure and build. # So, we use "OpenFST port for Windows" to build openfst with cmake in Windows. # Openfst is compiled with glog/gflags to avoid log and flag conflicts with log and flags in wenet/libtorch. # To build openfst with gflags and glog, we comment out some vars of {flags, log}.h and flags.cc. set(openfst_SOURCE_DIR ${fc_base}/openfst-src CACHE PATH "OpenFST source directory") - set(openfst_BINARY_DIR ${fc_base}/openfst-build CACHE PATH "OpenFST build directory") - set(openfst_PREFIX_DIR ${fc_base}/openfst-subbuild/openfst-populate-prefix CACHE PATH "OpenFST prefix directory") - if(NOT MSVC) - ExternalProject_Add(openfst - URL https://github.com/mjansche/openfst/archive/1.6.5.zip - URL_HASH SHA256=b720357a464f42e181d7e33f60867b54044007f50baedc8f4458a3926f4a5a78 - PREFIX ${openfst_PREFIX_DIR} - SOURCE_DIR ${openfst_SOURCE_DIR} - BINARY_DIR ${openfst_BINARY_DIR} - CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure ${CONFIG_FLAGS} --prefix=${openfst_PREFIX_DIR} - "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR} ${TORCH_CXX_FLAGS}" - "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}" - "LIBS=-lgflags_nothreads -lglog -lpthread" - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR} - BUILD_COMMAND make -j$(nproc) - ) - add_dependencies(openfst gflags glog) - link_directories(${openfst_PREFIX_DIR}/lib) - else() - add_compile_options(/W0 /wd4244 /wd4267) FetchContent_Declare(openfst URL https://github.com/kkm000/openfst/archive/refs/tags/win/1.6.5.1.tar.gz URL_HASH SHA256=02c49b559c3976a536876063369efc0e41ab374be1035918036474343877046e @@ -60,7 +32,6 @@ if(NOT ANDROID) FetchContent_MakeAvailable(openfst) add_dependencies(fst gflags glog) target_link_libraries(fst PUBLIC gflags_nothreads_static glog) - endif() include_directories(${openfst_SOURCE_DIR}/src/include) else() set(openfst_BINARY_DIR ${build_DIR}/wenet-openfst-android-1.0.2.aar/jni) diff --git a/runtime/libtorch/CMakeLists.txt b/runtime/libtorch/CMakeLists.txt index deb5de8773..7f8bd28abc 100644 --- a/runtime/libtorch/CMakeLists.txt +++ b/runtime/libtorch/CMakeLists.txt @@ -3,9 +3,8 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR) project(wenet VERSION 0.1) option(CXX11_ABI "whether to use CXX11_ABI libtorch" OFF) -option(FST_HAVE_BIN "whether to build fst binaries" OFF) +option(GRAPH_TOOLS "whether to build TLG graph tools" OFF) option(BUILD_TESTING "whether to build unit test" ON) -option(GRAPH_TOOLS "whether to build TLG graph tools" ON) option(GRPC "whether to build with gRPC" OFF) # TODO(Binbin Zhang): Change websocket to OFF since it depends on boost @@ -48,9 +47,6 @@ include_directories( # Build all libraries add_subdirectory(utils) -if(NOT MSVC) - add_dependencies(utils openfst) -endif() add_subdirectory(frontend) add_subdirectory(post_processor) add_subdirectory(kaldi) # kaldi: wfst based decoder diff --git a/runtime/raspberrypi/.gitignore b/runtime/raspberrypi/.gitignore new file mode 100644 index 0000000000..c6767241c3 --- /dev/null +++ b/runtime/raspberrypi/.gitignore @@ -0,0 +1,2 @@ +build/ +fc_base/ diff --git a/runtime/raspberrypi/CMakeLists.txt b/runtime/raspberrypi/CMakeLists.txt new file mode 120000 index 0000000000..492b84242b --- /dev/null +++ b/runtime/raspberrypi/CMakeLists.txt @@ -0,0 +1 @@ +../libtorch/CMakeLists.txt \ No newline at end of file diff --git a/runtime/raspberrypi/README.md b/runtime/raspberrypi/README.md new file mode 100644 index 0000000000..9cb1943e7e --- /dev/null +++ b/runtime/raspberrypi/README.md @@ -0,0 +1,52 @@ +# WeNet & Raspberry PI (Cross Compile) + +* Step 1. Install cross compile tools in the PC. + +``` sh +sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu +``` + +Or download, and install the binaries from: https://releases.linaro.org/components/toolchain/binaries/latest-7 + + +* Step 2. Export your experiment model to ONNX by https://github.com/wenet-e2e/wenet/blob/main/wenet/bin/export_onnx_cpu.py + +``` sh +exp=exp # Change it to your experiment dir +onnx_dir=onnx +python -m wenet.bin.export_onnx_cpu \ + --config $exp/train.yaml \ + --checkpoint $exp/final.pt \ + --chunk_size 16 \ + --output_dir $onnx_dir \ + --num_decoding_left_chunks -1 + +# When it finishes, you can find `encoder.onnx(.quant)`, `ctc.onnx(.quant)`, and `decoder.onnx(.quant)` in the $onnx_dir respectively. +# We use the quantified to speed up the inference, so rename it without the suffix `.quant` +``` + +* Step 3. Build. The build requires cmake 3.14 or above. and Send the binary and libraries to Raspberry PI. + +``` sh +cmake -B build -DONNX=ON -DTORCH=OFF -DWEBSOCKET=OFF -DGRPC=OFF -DCMAKE_TOOLCHAIN_FILE=toolchains/aarch64-linux-gnu.toolchain.cmake +cmake --build build +scp build/bin/decoder_main pi@xxx.xxx.xxx:/path/to/wenet +scp fc_base/onnxruntime-src/lib/libonnxruntime.so* pi@xxx.xxx.xxx:/path/to/wenet +``` + +* Step 4. Testing, the RTF(real time factor) is shown in Raspberry PI's console. + +``` sh +cd /path/to/wenet +export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH +export GLOG_logtostderr=1 +export GLOG_v=2 +wav_path=your_test_wav_path +onnx_dir=your_model_dir +units=units.txt # Change it to your model units path +./build/bin/decoder_main \ + --chunk_size 16 \ + --wav_path $wav_path \ + --onnx_dir $onnx_dir \ + --unit_path $units 2>&1 | tee log.txt +``` diff --git a/runtime/raspberrypi/api b/runtime/raspberrypi/api new file mode 120000 index 0000000000..5c1acaccc3 --- /dev/null +++ b/runtime/raspberrypi/api @@ -0,0 +1 @@ +../core/api \ No newline at end of file diff --git a/runtime/raspberrypi/bin b/runtime/raspberrypi/bin new file mode 120000 index 0000000000..938df72152 --- /dev/null +++ b/runtime/raspberrypi/bin @@ -0,0 +1 @@ +../core/bin \ No newline at end of file diff --git a/runtime/raspberrypi/cmake b/runtime/raspberrypi/cmake new file mode 120000 index 0000000000..17afee87dc --- /dev/null +++ b/runtime/raspberrypi/cmake @@ -0,0 +1 @@ +../core/cmake \ No newline at end of file diff --git a/runtime/raspberrypi/decoder b/runtime/raspberrypi/decoder new file mode 120000 index 0000000000..3088ea48b2 --- /dev/null +++ b/runtime/raspberrypi/decoder @@ -0,0 +1 @@ +../core/decoder \ No newline at end of file diff --git a/runtime/raspberrypi/frontend b/runtime/raspberrypi/frontend new file mode 120000 index 0000000000..0292335d13 --- /dev/null +++ b/runtime/raspberrypi/frontend @@ -0,0 +1 @@ +../core/frontend \ No newline at end of file diff --git a/runtime/raspberrypi/kaldi b/runtime/raspberrypi/kaldi new file mode 120000 index 0000000000..764a9d445d --- /dev/null +++ b/runtime/raspberrypi/kaldi @@ -0,0 +1 @@ +../core/kaldi \ No newline at end of file diff --git a/runtime/raspberrypi/patch b/runtime/raspberrypi/patch new file mode 120000 index 0000000000..69789fa5e4 --- /dev/null +++ b/runtime/raspberrypi/patch @@ -0,0 +1 @@ +../core/patch \ No newline at end of file diff --git a/runtime/raspberrypi/post_processor b/runtime/raspberrypi/post_processor new file mode 120000 index 0000000000..4e434a5cca --- /dev/null +++ b/runtime/raspberrypi/post_processor @@ -0,0 +1 @@ +../core/post_processor \ No newline at end of file diff --git a/runtime/raspberrypi/test b/runtime/raspberrypi/test new file mode 120000 index 0000000000..e60cf87a7f --- /dev/null +++ b/runtime/raspberrypi/test @@ -0,0 +1 @@ +../core/test \ No newline at end of file diff --git a/runtime/onnxruntime/toolchains b/runtime/raspberrypi/toolchains similarity index 100% rename from runtime/onnxruntime/toolchains rename to runtime/raspberrypi/toolchains diff --git a/runtime/raspberrypi/utils b/runtime/raspberrypi/utils new file mode 120000 index 0000000000..9e19e7af5a --- /dev/null +++ b/runtime/raspberrypi/utils @@ -0,0 +1 @@ +../core/utils \ No newline at end of file