Merge branch 'dev_infer' of https://github.com/alibaba-damo-academy/FunASR into dev_infer

docs/index.rst

@@ -68,10 +68,12 @@ Overview
    ./runtime/onnxruntime_python.md
    ./runtime/onnxruntime_cpp.md
    ./runtime/libtorch_python.md
-   ./runtime/grpc_python.md
-   ./runtime/grpc_cpp.md
+   ./runtime/html5.md
    ./runtime/websocket_python.md
    ./runtime/websocket_cpp.md
+   ./runtime/grpc_python.md
+   ./runtime/grpc_cpp.md
+
 
 .. toctree::
    :maxdepth: 1

docs/runtime/html5.md

@@ -0,0 +1 @@
+../../funasr/runtime/html5/readme.md

egs/aishell/paraformerbert/run.sh

@@ -146,7 +146,7 @@ if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
                 --data_dir ${feats_dir}/data \
                 --train_set ${train_set} \
                 --valid_set ${valid_set} \
-                --data_file_names "wav.scp,text,embed.scp" \
+                --data_file_names "wav.scp,text,embeds.scp" \
                 --cmvn_file ${feats_dir}/data/${train_set}/cmvn/cmvn.mvn \
                 --speed_perturb ${speed_perturb} \
                 --resume true \

egs/aishell2/paraformerbert/run.sh

@@ -147,7 +147,7 @@ if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
                 --data_dir ${feats_dir}/data \
                 --train_set ${train_set} \
                 --valid_set ${valid_set} \
-                --data_file_names "wav.scp,text,embed.scp" \
+                --data_file_names "wav.scp,text,embeds.scp" \
                 --cmvn_file ${feats_dir}/data/${train_set}/cmvn/cmvn.mvn \
                 --speed_perturb ${speed_perturb} \
                 --dataset_type $dataset_type \

egs/librispeech/conformer/conf/decode_asr_transformer.yaml → egs/librispeech/conformer/conf/decode_asr_transformer_ctc0.3_beam5.yaml

@@ -1,6 +1,6 @@
-beam_size: 10
+beam_size: 5
 penalty: 0.0
 maxlenratio: 0.0
 minlenratio: 0.0
-ctc_weight: 0.5
-lm_weight: 0.7
+ctc_weight: 0.3
+lm_weight: 0.0

egs/librispeech_100h/conformer/conf/decode_asr_transformer.yaml → egs/librispeech/conformer/conf/decode_asr_transformer_ctc0.3_beam60.yaml

@@ -1,6 +1,6 @@
-beam_size: 10
+beam_size: 60
 penalty: 0.0
 maxlenratio: 0.0
 minlenratio: 0.0
-ctc_weight: 0.5
-lm_weight: 0.7
+ctc_weight: 0.3
+lm_weight: 0.0

egs/librispeech/conformer/run.sh

@@ -53,8 +53,8 @@ test_sets="test_clean test_other dev_clean dev_other"
 asr_config=conf/train_asr_conformer.yaml
 model_dir="baseline_$(basename "${asr_config}" .yaml)_${lang}_${token_type}_${tag}"
 
-inference_config=conf/decode_asr_transformer.yaml
-#inference_config=conf/decode_asr_transformer_beam60_ctc0.3.yaml
+inference_config=conf/decode_asr_transformer_ctc0.3_beam5yaml
+#inference_config=conf/decode_asr_transformer_ctc0.3_beam60.yaml
 inference_asr_model=valid.acc.ave_10best.pb
 
 # you can set gpu num for decoding here

egs/librispeech_100h/conformer/conf/decode_asr_transformer_ctc0.3_beam1.yaml

@@ -0,0 +1,6 @@
+beam_size: 1
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.3
+lm_weight: 0.0

egs/librispeech_100h/conformer/conf/decode_asr_transformer_ctc0.3_beam20.yaml

@@ -0,0 +1,6 @@
+beam_size: 20
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.3
+lm_weight: 0.0

egs/librispeech_100h/conformer/conf/decode_asr_transformer_ctc0.3_beam5.yaml

@@ -0,0 +1,6 @@
+beam_size: 5
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.3
+lm_weight: 0.0

egs/librispeech_100h/conformer/run.sh

@@ -53,8 +53,9 @@ test_sets="test_clean test_other dev_clean dev_other"
 asr_config=conf/train_asr_conformer.yaml
 model_dir="baseline_$(basename "${asr_config}" .yaml)_${lang}_${token_type}_${tag}"
 
-inference_config=conf/decode_asr_transformer.yaml
-#inference_config=conf/decode_asr_transformer_beam60_ctc0.3.yaml
+#inference_config=conf/decode_asr_transformer_ctc0.3_beam1.yaml
+inference_config=conf/decode_asr_transformer_ctc0.3_beam5.yaml
+#inference_config=conf/decode_asr_transformer_ctc0.3_beam20.yaml
 inference_asr_model=valid.acc.ave_10best.pb
 
 # you can set gpu num for decoding here

egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vadrealtime-vocab272727/demo.py

@@ -9,7 +9,7 @@ logger.setLevel(logging.CRITICAL)
 inference_pipeline = pipeline(
     task=Tasks.punctuation,
     model='damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727',
-    output_dir="./tmp/"
+    model_revision = 'v1.0.2'
 )
 
 ##################text二进制数据#####################

funasr/export/test/test_onnx_punc_vadrealtime.py

@@ -12,7 +12,7 @@ if __name__ == '__main__':
         return {'inputs': np.ones((1, text_length), dtype=np.int64),
                 'text_lengths': np.array([text_length,], dtype=np.int32),
                 'vad_masks': np.ones((1, 1, text_length, text_length), dtype=np.float32),
-                'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :].astype(np.float32)
+                'sub_masks': np.ones((1, 1, text_length, text_length), dtype=np.float32),
                 }
 
     def _run(feed_dict):

funasr/runtime/html5/readme.md

@@ -9,70 +9,70 @@ pyOpenSSL
 ```
 
 ### javascript
-[html5录音](https://github.com/xiangyuecn/Recorder)
+[html5 recorder.js](https://github.com/xiangyuecn/Recorder)
 ```shell
 Recorder 
 ```
 
-### demo页面如下
-![img](https://github.com/alibaba-damo-academy/FunASR/blob/for-html5-demo/funasr/runtime/html5/demo.gif)
+### demo
+![img](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/html5/demo.gif)
 
-## 两种ws_server_online连接模式
-### 1)直接连接模式，浏览器https麦克风 --> html5 demo服务 --> js wss接口 --> wss asr online srv(证书生成请往后看)
+## wss or ws protocol for ws_server_online
+1) wss: browser microphone data --> html5 demo server --> js wss api --> wss asr online srv #for certificate generation just look back
 
-### 2)nginx中转，浏览器https麦克风 --> html5 demo服务 --> js wss接口 --> nginx服务 --> ws asr online srv
+2) ws: browser microphone data  --> html5 demo server --> js wss api --> nginx wss server --> ws asr online srv
 
-## 1.html5 demo服务启动
-### 启动html5服务，需要ssl证书(自己生成请往后看)
+## 1.html5 demo start
+### ssl certificate is required
 
 ```shell
 usage: h5Server.py [-h] [--host HOST] [--port PORT] [--certfile CERTFILE]
                    [--keyfile KEYFILE]
 python h5Server.py --port 1337
 ```
-## 2.启动ws or wss asr online srv
-[具体请看online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
-online asr提供两种ws和wss模式，wss模式可以直接启动，无需nginx中转。否则需要通过nginx将wss转发到该online asr的ws端口上
-### wss方式
+## 2.asr online srv start
+[detail for online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
+Online asr provides wss or ws way. if started in ws way, nginx is required for relay.
+### wss way, ssl certificate is required
 ```shell
 python ws_server_online.py --certfile server.crt --keyfile server.key  --port 5921
 ```
-### ws方式
+### ws way
 ```shell
 python ws_server_online.py  --port 5921
 ```
-## 3.修改wsconnecter.js里asr接口地址
-wsconnecter.js里配置online asr服务地址路径，这里配置的是wss端口
+## 3.modify asr address in wsconnecter.js according to your environment
+asr address in wsconnecter.js must be wss, just like
 var Uri = "wss://xxx:xxx/" 
 
-## 4.浏览器打开地址测试
-https://127.0.0.1:1337/static/index.html
+## 4.open browser to access html5 demo
+https://youraddress:port/static/index.html
 
 
 
 
-## 自行生成证书
-生成证书(注意这种证书并不能被所有浏览器认可，部分手动授权可以访问,最好使用其他认证的官方ssl证书)
+## certificate generation by yourself
+generated certificate may not suitable for all browsers due to security concerns. you'd better buy or download an authenticated ssl certificate from authorized agency.
 
 ```shell
-### 1)生成私钥，按照提示填写内容
+### 1) Generate a private key
 openssl genrsa -des3 -out server.key 1024
  
-### 2)生成csr文件 ，按照提示填写内容
+### 2) Generate a csr file
 openssl req -new -key server.key -out server.csr
  
-### 去掉pass
+### 3) Remove pass
 cp server.key server.key.org 
 openssl rsa -in server.key.org -out server.key
  
-### 生成crt文件，有效期1年（365天）
+### 4) Generated a crt file, valid for 1 year
 openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
 ```
 
-## nginx配置说明(了解的可以跳过)
-h5打开麦克风需要https协议，同时后端的asr websocket也必须是wss协议，如果[online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)以ws方式运行，我们可以通过nginx配置实现wss协议到ws协议的转换。
-
-### nginx转发配置示例
+## nginx configuration (you can skip it if you known)
+https and wss protocol are required by browsers when want to open microphone and websocket.  
+if [online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket) run in ws way, you should use nginx to convert wss to ws.
+### nginx wss->ws configuration example
 ```shell
 events {                                                                                                            [0/1548]
     worker_connections  1024;

funasr/runtime/html5/readme_cn.md

@@ -0,0 +1,111 @@
+# online asr demo for html5
+
+## requirement
+### python
+```shell
+flask
+gevent
+pyOpenSSL
+```
+
+### javascript
+[html5录音](https://github.com/xiangyuecn/Recorder)
+```shell
+Recorder 
+```
+
+### demo页面如下
+![img](https://github.com/alibaba-damo-academy/FunASR/blob/for-html5-demo/funasr/runtime/html5/demo.gif)
+
+## 两种ws_server_online连接模式
+### 1)直接连接模式，浏览器https麦克风 --> html5 demo服务 --> js wss接口 --> wss asr online srv(证书生成请往后看)
+
+### 2)nginx中转，浏览器https麦克风 --> html5 demo服务 --> js wss接口 --> nginx服务 --> ws asr online srv
+
+## 1.html5 demo服务启动
+### 启动html5服务，需要ssl证书(自己生成请往后看)
+
+```shell
+usage: h5Server.py [-h] [--host HOST] [--port PORT] [--certfile CERTFILE]
+                   [--keyfile KEYFILE]
+python h5Server.py --port 1337
+```
+## 2.启动ws or wss asr online srv
+[具体请看online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
+online asr提供两种ws和wss模式，wss模式可以直接启动，无需nginx中转。否则需要通过nginx将wss转发到该online asr的ws端口上
+### wss方式
+```shell
+python ws_server_online.py --certfile server.crt --keyfile server.key  --port 5921
+```
+### ws方式
+```shell
+python ws_server_online.py  --port 5921
+```
+## 3.修改wsconnecter.js里asr接口地址
+wsconnecter.js里配置online asr服务地址路径，这里配置的是wss端口
+var Uri = "wss://xxx:xxx/" 
+
+## 4.浏览器打开地址测试
+https://127.0.0.1:1337/static/index.html
+
+
+
+
+## 自行生成证书
+生成证书(注意这种证书并不能被所有浏览器认可，部分手动授权可以访问,最好使用其他认证的官方ssl证书)
+
+```shell
+### 1)生成私钥，按照提示填写内容
+openssl genrsa -des3 -out server.key 1024
+ 
+### 2)生成csr文件 ，按照提示填写内容
+openssl req -new -key server.key -out server.csr
+ 
+### 去掉pass
+cp server.key server.key.org 
+openssl rsa -in server.key.org -out server.key
+ 
+### 生成crt文件，有效期1年（365天）
+openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+```
+
+## nginx配置说明(了解的可以跳过)
+h5打开麦克风需要https协议，同时后端的asr websocket也必须是wss协议，如果[online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)以ws方式运行，我们可以通过nginx配置实现wss协议到ws协议的转换。
+
+### nginx转发配置示例
+```shell
+events {                                                                                                            [0/1548]
+    worker_connections  1024;
+    accept_mutex on;
+  }
+http {
+  error_log  error.log;
+  access_log  access.log;
+  server {
+
+    listen 5921 ssl http2;  # nginx listen port for wss
+    server_name www.test.com;
+
+    ssl_certificate     /funasr/server.crt;
+    ssl_certificate_key /funasr/server.key;
+    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+    ssl_ciphers         HIGH:!aNULL:!MD5;
+
+    location /wss/ {
+
+
+      proxy_pass http://127.0.0.1:1111/;  # asr online model ws address and port
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+      proxy_read_timeout 600s;
+
+    }
+  }
+```
+### 修改wsconnecter.js里asr接口地址
+wsconnecter.js里配置online asr服务地址路径，这里配置的是wss端口
+var Uri = "wss://xxx:xxx/wss/" 
+## Acknowledge
+1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR).
+2. We acknowledge [AiHealthx](http://www.aihealthx.com/) for contributing the html5 demo.

funasr/runtime/html5/static/wsconnecter.js

@@ -5,7 +5,7 @@
 /* 2021-2023 by zhaoming,mali aihealthx.com */
 
 function WebSocketConnectMethod( config ) { //定义socket连接方法类
-	var Uri = "wss://111.205.137.58:5821/wss/" //设置wss asr online接口地址 如 wss://X.X.X.X:port/wss/
+    var Uri = "wss://30.220.136.139:5921/"  //	var Uri = "wss://30.221.177.46:5921/" //设置wss asr online接口地址 如 wss://X.X.X.X:port/wss/
 	var speechSokt;
 	var connKeeperID;
 	

funasr/runtime/onnxruntime/include/vad-model.h

@@ -11,15 +11,11 @@ class VadModel {
   public:
     virtual ~VadModel(){};
     virtual void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num)=0;
-    virtual std::vector<std::vector<int>> Infer(const std::vector<float> &waves)=0;
+    virtual std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true)=0;
     virtual void ReadModel(const char* vad_model)=0;
     virtual void LoadConfigFromYaml(const char* filename)=0;
     virtual void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
-                    const std::vector<float> &waves)=0;
-    virtual void LfrCmvn(std::vector<std::vector<float>> &vad_feats)=0;
-    virtual void Forward(
-            const std::vector<std::vector<float>> &chunk_feats,
-            std::vector<std::vector<float>> *out_prob)=0;
+                    std::vector<float> &waves)=0;
     virtual void LoadCmvn(const char *filename)=0;
     virtual void InitCache()=0;
 };

funasr/runtime/onnxruntime/readme.md

@@ -127,6 +127,8 @@ For example:
 ### funasr-onnx-offline-rtf
 ```shell
 ./funasr-onnx-offline-rtf     --model-dir <string> [--quantize <string>]
+                              [--vad-dir <string>] [--vad-quant <string>]
+                              [--punc-dir <string>] [--punc-quant <string>]
                               --wav-path <string> --thread-num <int32_t>
                               [--] [--version] [-h]
 Where:
@@ -136,6 +138,17 @@ Where:
      (required)  the model path, which contains model.onnx, config.yaml, am.mvn
    --quantize <string>
      false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir
+
+   --vad-dir <string>
+     the vad model path, which contains model.onnx, vad.yaml, vad.mvn
+   --vad-quant <string>
+     false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir
+
+   --punc-dir <string>
+     the punc model path, which contains model.onnx, punc.yaml
+   --punc-quant <string>
+     false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir
+     
    --wav-path <string>
      (required)  the input could be: 
       wav_path, e.g.: asr_example.wav;

funasr/runtime/onnxruntime/src/fsmn-vad.cpp

@@ -162,17 +162,21 @@ void FsmnVad::Forward(
     }
   
     // get 4 caches outputs,each size is 128*19
-    for (int i = 1; i < 5; i++) {
-      float* data = vad_ort_outputs[i].GetTensorMutableData<float>();
-      memcpy(in_cache_[i-1].data(), data, sizeof(float) * 128*19);
-    }
+    // for (int i = 1; i < 5; i++) {
+    //   float* data = vad_ort_outputs[i].GetTensorMutableData<float>();
+    //   memcpy(in_cache_[i-1].data(), data, sizeof(float) * 128*19);
+    // }
 }
 
 void FsmnVad::FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
-                         const std::vector<float> &waves) {
+                         std::vector<float> &waves) {
     knf::OnlineFbank fbank(fbank_opts);
 
-    fbank.AcceptWaveform(sample_rate, &waves[0], waves.size());
+    std::vector<float> buf(waves.size());
+    for (int32_t i = 0; i != waves.size(); ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank.AcceptWaveform(sample_rate, buf.data(), buf.size());
     int32_t frames = fbank.NumFramesReady();
     for (int32_t i = 0; i != frames; ++i) {
         const float *frame = fbank.GetFrame(i);
@@ -267,7 +271,7 @@ void FsmnVad::LfrCmvn(std::vector<std::vector<float>> &vad_feats) {
 }
 
 std::vector<std::vector<int>>
-FsmnVad::Infer(const std::vector<float> &waves) {
+FsmnVad::Infer(std::vector<float> &waves, bool input_finished) {
     std::vector<std::vector<float>> vad_feats;
     std::vector<std::vector<float>> vad_probs;
     FbankKaldi(vad_sample_rate_, vad_feats, waves);

funasr/runtime/onnxruntime/src/fsmn-vad.h

@@ -21,7 +21,7 @@ public:
     ~FsmnVad();
     void Test();
     void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num);
-    std::vector<std::vector<int>> Infer(const std::vector<float> &waves);
+    std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true);
     void Reset();
 
 private:
@@ -34,7 +34,7 @@ private:
             std::vector<const char *> *in_names, std::vector<const char *> *out_names);
 
     void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
-                    const std::vector<float> &waves);
+                    std::vector<float> &waves);
 
     void LfrCmvn(std::vector<std::vector<float>> &vad_feats);
 

funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp

@@ -39,7 +39,7 @@ void runReg(FUNASR_HANDLE asr_handle, vector<string> wav_list,
     // warm up
     for (size_t i = 0; i < 1; i++)
     {
-        FUNASR_RESULT result=FunASRInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, 16000);
+        FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, 16000);
     }
 
     while (true) {
@@ -50,7 +50,7 @@ void runReg(FUNASR_HANDLE asr_handle, vector<string> wav_list,
         }
 
         gettimeofday(&start, NULL);
-        FUNASR_RESULT result=FunASRInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, 16000);
+        FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, 16000);
 
         gettimeofday(&end, NULL);
         seconds = (end.tv_sec - start.tv_sec);
@@ -102,12 +102,20 @@ int main(int argc, char *argv[])
     TCLAP::CmdLine cmd("funasr-onnx-offline-rtf", ' ', "1.0");
     TCLAP::ValueArg<std::string>    model_dir("", MODEL_DIR, "the model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
     TCLAP::ValueArg<std::string>    quantize("", QUANTIZE, "false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "false", "string");
+    TCLAP::ValueArg<std::string>    vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+    TCLAP::ValueArg<std::string>    vad_quant("", VAD_QUANT, "false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "false", "string");
+    TCLAP::ValueArg<std::string>    punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+    TCLAP::ValueArg<std::string>    punc_quant("", PUNC_QUANT, "false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "false", "string");
 
     TCLAP::ValueArg<std::string> wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
     TCLAP::ValueArg<std::int32_t> thread_num("", THREAD_NUM, "multi-thread num for rtf", true, 0, "int32_t");
 
     cmd.add(model_dir);
     cmd.add(quantize);
+    cmd.add(vad_dir);
+    cmd.add(vad_quant);
+    cmd.add(punc_dir);
+    cmd.add(punc_quant);
     cmd.add(wav_path);
     cmd.add(thread_num);
     cmd.parse(argc, argv);
@@ -115,11 +123,15 @@ int main(int argc, char *argv[])
     std::map<std::string, std::string> model_path;
     GetValue(model_dir, MODEL_DIR, model_path);
     GetValue(quantize, QUANTIZE, model_path);
+    GetValue(vad_dir, VAD_DIR, model_path);
+    GetValue(vad_quant, VAD_QUANT, model_path);
+    GetValue(punc_dir, PUNC_DIR, model_path);
+    GetValue(punc_quant, PUNC_QUANT, model_path);
     GetValue(wav_path, WAV_PATH, model_path);
 
     struct timeval start, end;
     gettimeofday(&start, NULL);
-    FUNASR_HANDLE asr_handle=FunASRInit(model_path, 1);
+    FUNASR_HANDLE asr_handle=FunOfflineInit(model_path, 1);
 
     if (!asr_handle)
     {
@@ -132,7 +144,7 @@ int main(int argc, char *argv[])
     long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
     LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
 
-    // read wav_scp
+    // read wav_path
     vector<string> wav_list;
     string wav_path_ = model_path.at(WAV_PATH);
     if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
@@ -179,6 +191,6 @@ int main(int argc, char *argv[])
     LOG(INFO) << "total_rtf " << (double)total_time/ (total_length*1000000);
     LOG(INFO) << "speedup " << 1.0/((double)total_time/ (total_length*1000000));
 
-    FunASRUninit(asr_handle);
+    FunOfflineUninit(asr_handle);
     return 0;
 }

funasr/runtime/onnxruntime/src/paraformer.cpp

@@ -69,7 +69,11 @@ void Paraformer::Reset()
 
 vector<float> Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) {
     knf::OnlineFbank fbank_(fbank_opts);
-    fbank_.AcceptWaveform(sample_rate, waves, len);
+    std::vector<float> buf(len);
+    for (int32_t i = 0; i != len; ++i) {
+        buf[i] = waves[i] * 32768;
+    }
+    fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
     //fbank_->InputFinished();
     int32_t frames = fbank_.NumFramesReady();
     int32_t feature_dim = fbank_opts.mel_opts.num_bins;

funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py

@@ -186,11 +186,12 @@ class CT_Transformer_VadRealtime(CT_Transformer):
             mini_sentence = cache_sent + mini_sentence
             mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0,dtype='int32')
             text_length = len(mini_sentence_id)
+            vad_mask = self.vad_mask(text_length, len(cache))[None, None, :, :].astype(np.float32)
             data = {
                 "input": mini_sentence_id[None,:],
                 "text_lengths": np.array([text_length], dtype='int32'),
-                "vad_mask": self.vad_mask(text_length, len(cache))[None, None, :, :].astype(np.float32),
-                "sub_masks": np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :].astype(np.float32)
+                "vad_mask": vad_mask,
+                "sub_masks": vad_mask
             }
             try:
                 outputs = self.infer(data['input'], data['text_lengths'], data['vad_mask'], data["sub_masks"])

funasr/runtime/python/websocket/ws_server_online.py

@@ -32,15 +32,29 @@ inference_pipeline_asr_online = pipeline(
 	ncpu=args.ncpu,
 	model_revision='v1.0.4')
 
+# vad
+inference_pipeline_vad = pipeline(
+    task=Tasks.voice_activity_detection,
+    model=args.vad_model,
+    model_revision=None,
+    output_dir=None,
+    batch_size=1,
+    mode='online',
+    ngpu=args.ngpu,
+    ncpu=1,
+)
+
 print("model loaded")
 
 
 
 async def ws_serve(websocket, path):
+	frames = []
 	frames_asr_online = []
 	global websocket_users
 	websocket_users.add(websocket)
 	websocket.param_dict_asr_online = {"cache": dict()}
+	websocket.param_dict_vad = {'in_cache': dict()}
 	websocket.wav_name = "microphone"
 	print("new user connected",flush=True)
 	try:
@@ -53,9 +67,10 @@ async def ws_serve(websocket, path):
 				if "is_speaking" in messagejson:
 					websocket.is_speaking = messagejson["is_speaking"]
 					websocket.param_dict_asr_online["is_final"] = not websocket.is_speaking
+					websocket.param_dict_vad["is_final"] = not websocket.is_speaking
 					# need to fire engine manually if no data received any more
 					if not websocket.is_speaking:
-						await async_asr_online(websocket,b"")
+						await async_asr_online(websocket, b"")
 				if "chunk_interval" in messagejson:
 					websocket.chunk_interval=messagejson["chunk_interval"]
 				if "wav_name" in messagejson:
@@ -64,14 +79,18 @@ async def ws_serve(websocket, path):
 					websocket.param_dict_asr_online["chunk_size"] = messagejson["chunk_size"]
 			# if has bytes in buffer or message is bytes
 			if len(frames_asr_online) > 0 or not isinstance(message, str):
-				if not isinstance(message,str):
+				if not isinstance(message, str):
 					frames_asr_online.append(message)
+					# frames.append(message)
+					# duration_ms = len(message) // 32
+					# websocket.vad_pre_idx += duration_ms
+					speech_start_i, speech_end_i = await async_vad(websocket, message)
+					websocket.is_speaking = not speech_end_i
+					
 				if len(frames_asr_online) % websocket.chunk_interval == 0 or not websocket.is_speaking:
+					websocket.param_dict_asr_online["is_final"] = not websocket.is_speaking
 					audio_in = b"".join(frames_asr_online)
-					# if not websocket.is_speaking:
-						#padding 0.5s at end gurantee that asr engine can fire out last word
-						# audio_in=audio_in+b''.join(np.zeros(int(16000*0.5),dtype=np.int16))
-					await async_asr_online(websocket,audio_in)
+					await async_asr_online(websocket, audio_in)
 					frames_asr_online = []
 	
 	
@@ -85,7 +104,7 @@ async def ws_serve(websocket, path):
 
 
 async def async_asr_online(websocket,audio_in):
-	if len(audio_in) >=0:
+	if len(audio_in) >= 0:
 		audio_in = load_bytes(audio_in)
 		rec_result = inference_pipeline_asr_online(audio_in=audio_in,
 		                                           param_dict=websocket.param_dict_asr_online)
@@ -97,16 +116,30 @@ async def async_asr_online(websocket,audio_in):
 				await websocket.send(message)
 
 
-if len(args.certfile)>0:
-  ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-
-  # Generate with Lets Encrypt, copied to this location, chown to current user and 400 permissions
-  ssl_cert = args.certfile
-  ssl_key = args.keyfile
+async def async_vad(websocket, audio_in):
+	segments_result = inference_pipeline_vad(audio_in=audio_in, param_dict=websocket.param_dict_vad)
+	
+	speech_start = False
+	speech_end = False
+	
+	if len(segments_result) == 0 or len(segments_result["text"]) > 1:
+		return speech_start, speech_end
+	if segments_result["text"][0][0] != -1:
+		speech_start = segments_result["text"][0][0]
+	if segments_result["text"][0][1] != -1:
+		speech_end = True
+	return speech_start, speech_end
 
-  ssl_context.load_cert_chain(ssl_cert, keyfile=ssl_key)
-  start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None,ssl=ssl_context)
+if len(args.certfile)>0:
+	ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+	
+	# Generate with Lets Encrypt, copied to this location, chown to current user and 400 permissions
+	ssl_cert = args.certfile
+	ssl_key = args.keyfile
+	
+	ssl_context.load_cert_chain(ssl_cert, keyfile=ssl_key)
+	start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None,ssl=ssl_context)
 else:
-  start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None)
+	start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None)
 asyncio.get_event_loop().run_until_complete(start_server)
 asyncio.get_event_loop().run_forever()