debugging

funasr/bin/asr_inference_paraformer.py

@@ -43,6 +43,7 @@ from funasr.models.frontend.wav_frontend import WavFrontend
 from funasr.models.e2e_asr_paraformer import BiCifParaformer, ContextualParaformer
 from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_export
 from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
+from funasr.bin.tp_inference import SpeechText2Timestamp
 
 
 class Speech2Text:
@@ -540,7 +541,8 @@ def inference(
         ngram_weight: float = 0.9,
         nbest: int = 1,
         num_workers: int = 1,
-
+        timestamp_infer_config: Union[Path, str] = None,
+        timestamp_model_file: Union[Path, str] = None,
         **kwargs,
 ):
     inference_pipeline = inference_modelscope(
@@ -604,6 +606,8 @@ def inference_modelscope(
         nbest: int = 1,
         num_workers: int = 1,
         output_dir: Optional[str] = None,
+        timestamp_infer_config: Union[Path, str] = None,
+        timestamp_model_file: Union[Path, str] = None,
         param_dict: dict = None,
         **kwargs,
 ):
@@ -661,6 +665,15 @@ def inference_modelscope(
     else:
         speech2text = Speech2Text(**speech2text_kwargs)
 
+    if timestamp_model_file is not None:
+        speechtext2timestamp = SpeechText2Timestamp(
+            timestamp_cmvn_file=cmvn_file,
+            timestamp_model_file=timestamp_model_file,
+            timestamp_infer_config=timestamp_infer_config,
+        )
+    else:
+        speechtext2timestamp = None
+
     def _forward(
             data_path_and_name_and_type,
             raw_inputs: Union[np.ndarray, torch.Tensor] = None,
@@ -743,8 +756,16 @@ def inference_modelscope(
 
                 key = keys[batch_id]
                 for n, result in zip(range(1, nbest + 1), result):
+                    # import pdb; pdb.set_trace()
                     text, token, token_int, hyp = result[0], result[1], result[2], result[3]
                     time_stamp = None if len(result) < 5 else result[4]
+                    # conduct timestamp prediction here
+                    if time_stamp is None and speechtext2timestamp:
+                        ts_batch = {}
+                        ts_batch['speech'] = batch['speech'][batch_id].squeeze(0)
+                        ts_batch['speech_lengths'] = torch.tensor([batch['speech_lengths'][batch_id]])
+                        ts_batch['text_lengths'] = torch.tensor([len(token)])
+                        import pdb; pdb.set_trace()
                     # Create a directory: outdir/{n}best_recog
                     if writer is not None:
                         ibest_writer = writer[f"{n}best_recog"]

funasr/bin/asr_inference_paraformer_vad_punc.py

@@ -674,6 +674,7 @@ def inference_modelscope(
                         ibest_writer["time_stamp"][key] = "{}".format(time_stamp_postprocessed)
 
                 logging.info("decoding, utt: {}, predictions: {}".format(key, text_postprocessed_punc))
+        import pdb; pdb.set_trace()
         return asr_result_list
 
     return _forward

test.py

@@ -0,0 +1,28 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+'''
+inference_pipeline = pipeline(
+    task=Tasks.auto_speech_recognition,
+    model='damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
+    timestamp_model='damo/speech_timestamp_prediction-v1-16k-offline',
+    timestamp_model_revision='v1.0.3',
+    )
+
+rec_result = inference_pipeline(audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav')
+print(rec_result)
+'''
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+inference_pipeline = pipeline(
+    task=Tasks.auto_speech_recognition,
+    model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
+    vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',
+    vad_model_revision="v1.1.8",
+    punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
+    punc_model_revision="v1.1.6")
+
+rec_result = inference_pipeline(audio_in='/Users/shixian/Downloads/test.wav')
+print(rec_result)