paraformer long batch infer

egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/demo.py

@@ -2,14 +2,15 @@ from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 
 if __name__ == '__main__':
-    audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+    audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav'
     output_dir = None
     inference_pipeline = pipeline(
         task=Tasks.auto_speech_recognition,
         model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
         vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',
         punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
-        output_dir=output_dir
+        output_dir=output_dir,
+        batch_size=8,
     )
     rec_result = inference_pipeline(audio_in=audio_in)
     print(rec_result)

funasr/bin/asr_inference_paraformer.py

@@ -358,160 +358,6 @@ class Speech2Text:
             hotword_list = None
         return hotword_list
 
-class Speech2TextExport:
-    """Speech2TextExport class
-
-    """
-
-    def __init__(
-            self,
-            asr_train_config: Union[Path, str] = None,
-            asr_model_file: Union[Path, str] = None,
-            cmvn_file: Union[Path, str] = None,
-            lm_train_config: Union[Path, str] = None,
-            lm_file: Union[Path, str] = None,
-            token_type: str = None,
-            bpemodel: str = None,
-            device: str = "cpu",
-            maxlenratio: float = 0.0,
-            minlenratio: float = 0.0,
-            dtype: str = "float32",
-            beam_size: int = 20,
-            ctc_weight: float = 0.5,
-            lm_weight: float = 1.0,
-            ngram_weight: float = 0.9,
-            penalty: float = 0.0,
-            nbest: int = 1,
-            frontend_conf: dict = None,
-            hotword_list_or_file: str = None,
-            **kwargs,
-    ):
-
-        # 1. Build ASR model
-        asr_model, asr_train_args = ASRTask.build_model_from_file(
-            asr_train_config, asr_model_file, cmvn_file, device
-        )
-        frontend = None
-        if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
-            frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
-
-        logging.info("asr_model: {}".format(asr_model))
-        logging.info("asr_train_args: {}".format(asr_train_args))
-        asr_model.to(dtype=getattr(torch, dtype)).eval()
-
-        token_list = asr_model.token_list
-
-
-
-        logging.info(f"Decoding device={device}, dtype={dtype}")
-
-        # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
-        if token_type is None:
-            token_type = asr_train_args.token_type
-        if bpemodel is None:
-            bpemodel = asr_train_args.bpemodel
-
-        if token_type is None:
-            tokenizer = None
-        elif token_type == "bpe":
-            if bpemodel is not None:
-                tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel)
-            else:
-                tokenizer = None
-        else:
-            tokenizer = build_tokenizer(token_type=token_type)
-        converter = TokenIDConverter(token_list=token_list)
-        logging.info(f"Text tokenizer: {tokenizer}")
-
-        # self.asr_model = asr_model
-        self.asr_train_args = asr_train_args
-        self.converter = converter
-        self.tokenizer = tokenizer
-
-        self.device = device
-        self.dtype = dtype
-        self.nbest = nbest
-        self.frontend = frontend
-
-        model = Paraformer_export(asr_model, onnx=False)
-        self.asr_model = model
-        
-    @torch.no_grad()
-    def __call__(
-            self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
-    ):
-        """Inference
-
-        Args:
-                speech: Input speech data
-        Returns:
-                text, token, token_int, hyp
-
-        """
-        assert check_argument_types()
-
-        # Input as audio signal
-        if isinstance(speech, np.ndarray):
-            speech = torch.tensor(speech)
-
-        if self.frontend is not None:
-            feats, feats_len = self.frontend.forward(speech, speech_lengths)
-            feats = to_device(feats, device=self.device)
-            feats_len = feats_len.int()
-            self.asr_model.frontend = None
-        else:
-            feats = speech
-            feats_len = speech_lengths
-
-        enc_len_batch_total = feats_len.sum()
-        lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
-        batch = {"speech": feats, "speech_lengths": feats_len}
-
-        # a. To device
-        batch = to_device(batch, device=self.device)
-
-        decoder_outs = self.asr_model(**batch)
-        decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
-        
-        results = []
-        b, n, d = decoder_out.size()
-        for i in range(b):
-            am_scores = decoder_out[i, :ys_pad_lens[i], :]
-
-            yseq = am_scores.argmax(dim=-1)
-            score = am_scores.max(dim=-1)[0]
-            score = torch.sum(score, dim=-1)
-            # pad with mask tokens to ensure compatibility with sos/eos tokens
-            yseq = torch.tensor(
-                yseq.tolist(), device=yseq.device
-            )
-            nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
-
-            for hyp in nbest_hyps:
-                assert isinstance(hyp, (Hypothesis)), type(hyp)
-
-                # remove sos/eos and get results
-                last_pos = -1
-                if isinstance(hyp.yseq, list):
-                    token_int = hyp.yseq[1:last_pos]
-                else:
-                    token_int = hyp.yseq[1:last_pos].tolist()
-
-                # remove blank symbol id, which is assumed to be 0
-                token_int = list(filter(lambda x: x != 0 and x != 2, token_int))
-
-                # Change integer-ids to tokens
-                token = self.converter.ids2tokens(token_int)
-
-                if self.tokenizer is not None:
-                    text = self.tokenizer.tokens2text(token)
-                else:
-                    text = None
-
-                results.append((text, token, token_int, hyp, enc_len_batch_total, lfr_factor))
-
-        return results
-
 
 def inference(
         maxlenratio: float,
@@ -665,10 +511,8 @@ def inference_modelscope(
         nbest=nbest,
         hotword_list_or_file=hotword_list_or_file,
     )
-    if export_mode:
-        speech2text = Speech2TextExport(**speech2text_kwargs)
-    else:
-        speech2text = Speech2Text(**speech2text_kwargs)
+
+    speech2text = Speech2Text(**speech2text_kwargs)
 
     if timestamp_model_file is not None:
         speechtext2timestamp = SpeechText2Timestamp(

funasr/bin/asr_inference_paraformer_vad_punc.py

@@ -47,327 +47,323 @@ from funasr.bin.vad_inference import Speech2VadSegment
 from funasr.utils.timestamp_tools import time_stamp_sentence, ts_prediction_lfr6_standard
 from funasr.bin.punctuation_infer import Text2Punc
 from funasr.models.e2e_asr_paraformer import BiCifParaformer, ContextualParaformer
-
-
-header_colors = '\033[95m'
-end_colors = '\033[0m'
-
-
-class Speech2Text:
-    """Speech2Text class
-
-    Examples:
-            >>> import soundfile
-            >>> speech2text = Speech2Text("asr_config.yml", "asr.pb")
-            >>> audio, rate = soundfile.read("speech.wav")
-            >>> speech2text(audio)
-            [(text, token, token_int, hypothesis object), ...]
-
-    """
-
-    def __init__(
-            self,
-            asr_train_config: Union[Path, str] = None,
-            asr_model_file: Union[Path, str] = None,
-            cmvn_file: Union[Path, str] = None,
-            lm_train_config: Union[Path, str] = None,
-            lm_file: Union[Path, str] = None,
-            token_type: str = None,
-            bpemodel: str = None,
-            device: str = "cpu",
-            maxlenratio: float = 0.0,
-            minlenratio: float = 0.0,
-            dtype: str = "float32",
-            beam_size: int = 20,
-            ctc_weight: float = 0.5,
-            lm_weight: float = 1.0,
-            ngram_weight: float = 0.9,
-            penalty: float = 0.0,
-            nbest: int = 1,
-            frontend_conf: dict = None,
-            hotword_list_or_file: str = None,
-            **kwargs,
-    ):
-        assert check_argument_types()
-
-        # 1. Build ASR model
-        scorers = {}
-        asr_model, asr_train_args = ASRTask.build_model_from_file(
-            asr_train_config, asr_model_file, cmvn_file=cmvn_file, device=device
-        )
-        frontend = None
-        if asr_model.frontend is not None and asr_train_args.frontend_conf is not None:
-            frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
-
-        # logging.info("asr_model: {}".format(asr_model))
-        # logging.info("asr_train_args: {}".format(asr_train_args))
-        asr_model.to(dtype=getattr(torch, dtype)).eval()
-
-        if asr_model.ctc != None:
-            ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
-            scorers.update(
-                ctc=ctc
-            )
-        token_list = asr_model.token_list
-        scorers.update(
-            length_bonus=LengthBonus(len(token_list)),
-        )
-
-        # 2. Build Language model
-        if lm_train_config is not None:
-            lm, lm_train_args = LMTask.build_model_from_file(
-                lm_train_config, lm_file, device
-            )
-            scorers["lm"] = lm.lm
-
-        # 3. Build ngram model
-        # ngram is not supported now
-        ngram = None
-        scorers["ngram"] = ngram
-
-        # 4. Build BeamSearch object
-        # transducer is not supported now
-        beam_search_transducer = None
-
-        weights = dict(
-            decoder=1.0 - ctc_weight,
-            ctc=ctc_weight,
-            lm=lm_weight,
-            ngram=ngram_weight,
-            length_bonus=penalty,
-        )
-        beam_search = BeamSearch(
-            beam_size=beam_size,
-            weights=weights,
-            scorers=scorers,
-            sos=asr_model.sos,
-            eos=asr_model.eos,
-            vocab_size=len(token_list),
-            token_list=token_list,
-            pre_beam_score_key=None if ctc_weight == 1.0 else "full",
-        )
-
-        beam_search.to(device=device, dtype=getattr(torch, dtype)).eval()
-        for scorer in scorers.values():
-            if isinstance(scorer, torch.nn.Module):
-                scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
-
-        logging.info(f"Decoding device={device}, dtype={dtype}")
-
-        # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
-        if token_type is None:
-            token_type = asr_train_args.token_type
-        if bpemodel is None:
-            bpemodel = asr_train_args.bpemodel
-
-        if token_type is None:
-            tokenizer = None
-        elif token_type == "bpe":
-            if bpemodel is not None:
-                tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel)
-            else:
-                tokenizer = None
-        else:
-            tokenizer = build_tokenizer(token_type=token_type)
-        converter = TokenIDConverter(token_list=token_list)
-        logging.info(f"Text tokenizer: {tokenizer}")
-
-        self.asr_model = asr_model
-        self.asr_train_args = asr_train_args
-        self.converter = converter
-        self.tokenizer = tokenizer
-
-        # 6. [Optional] Build hotword list from str, local file or url
-        self.hotword_list = None
-        self.hotword_list = self.generate_hotwords_list(hotword_list_or_file)
-
-        is_use_lm = lm_weight != 0.0 and lm_file is not None
-        if (ctc_weight == 0.0 or asr_model.ctc == None) and not is_use_lm:
-            beam_search = None
-        self.beam_search = beam_search
-        logging.info(f"Beam_search: {self.beam_search}")
-        self.beam_search_transducer = beam_search_transducer
-        self.maxlenratio = maxlenratio
-        self.minlenratio = minlenratio
-        self.device = device
-        self.dtype = dtype
-        self.nbest = nbest
-        self.frontend = frontend
-        self.encoder_downsampling_factor = 1
-        if asr_train_args.encoder_conf["input_layer"] == "conv2d":
-            self.encoder_downsampling_factor = 4
-
-    @torch.no_grad()
-    def __call__(
-            self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None,
-            begin_time: int = 0, end_time: int = None,
-    ):
-        """Inference
-
-        Args:
-                speech: Input speech data
-        Returns:
-                text, token, token_int, hyp
-
-        """
-        assert check_argument_types()
-
-        # Input as audio signal
-        if isinstance(speech, np.ndarray):
-            speech = torch.tensor(speech)
-
-        if self.frontend is not None:
-            # feats, feats_len = self.frontend.forward(speech, speech_lengths)
-            # fbanks, fbanks_len = self.frontend.forward_fbank(speech, speech_lengths)
-            feats, feats_len = self.frontend.forward_lfr_cmvn(speech, speech_lengths)
-            feats = to_device(feats, device=self.device)
-            feats_len = feats_len.int()
-            self.asr_model.frontend = None
-        else:
-            feats = speech
-            feats_len = speech_lengths
-        lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
-        batch = {"speech": feats, "speech_lengths": feats_len}
-
-        # a. To device
-        batch = to_device(batch, device=self.device)
-
-        # b. Forward Encoder
-        enc, enc_len = self.asr_model.encode(**batch)
-        if isinstance(enc, tuple):
-            enc = enc[0]
-        # assert len(enc) == 1, len(enc)
-        enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
-
-        predictor_outs = self.asr_model.calc_predictor(enc, enc_len)
-        pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = predictor_outs[0], predictor_outs[1], \
-                                                                        predictor_outs[2], predictor_outs[3]
-        pre_token_length = pre_token_length.round().long()
-        if torch.max(pre_token_length) < 1:
-            return []
-
-        if not isinstance(self.asr_model, ContextualParaformer):
-            if self.hotword_list:
-                logging.warning("Hotword is given but asr model is not a ContextualParaformer.")
-            decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
-            decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
-        else:
-            decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length, hw_list=self.hotword_list)
-            decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
-
-        if isinstance(self.asr_model, BiCifParaformer):
-            _, _, us_alphas, us_peaks = self.asr_model.calc_predictor_timestamp(enc, enc_len,
-                                                                                   pre_token_length)  # test no bias cif2
-
-        results = []
-        b, n, d = decoder_out.size()
-        for i in range(b):
-            x = enc[i, :enc_len[i], :]
-            am_scores = decoder_out[i, :pre_token_length[i], :]
-            if self.beam_search is not None:
-                nbest_hyps = self.beam_search(
-                    x=x, am_scores=am_scores, maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
-                )
-
-                nbest_hyps = nbest_hyps[: self.nbest]
-            else:
-                yseq = am_scores.argmax(dim=-1)
-                score = am_scores.max(dim=-1)[0]
-                score = torch.sum(score, dim=-1)
-                # pad with mask tokens to ensure compatibility with sos/eos tokens
-                yseq = torch.tensor(
-                    [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
-                )
-                nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
-
-            for hyp in nbest_hyps:
-                assert isinstance(hyp, (Hypothesis)), type(hyp)
-
-                # remove sos/eos and get results
-                last_pos = -1
-                if isinstance(hyp.yseq, list):
-                    token_int = hyp.yseq[1:last_pos]
-                else:
-                    token_int = hyp.yseq[1:last_pos].tolist()
-
-                # remove blank symbol id, which is assumed to be 0
-                token_int = list(filter(lambda x: x != 0 and x != 2, token_int))
-                if len(token_int) == 0:
-                    continue
-
-                # Change integer-ids to tokens
-                token = self.converter.ids2tokens(token_int)
-
-                if self.tokenizer is not None:
-                    text = self.tokenizer.tokens2text(token)
-                else:
-                    text = None
-
-                if isinstance(self.asr_model, BiCifParaformer):
-                    _, timestamp = ts_prediction_lfr6_standard(us_alphas[i], 
-                                                            us_peaks[i], 
-                                                            copy.copy(token), 
-                                                            vad_offset=begin_time)
-                    results.append((text, token, token_int, timestamp, enc_len_batch_total, lfr_factor))
-                else:
-                    results.append((text, token, token_int, enc_len_batch_total, lfr_factor))
-
-        # assert check_return_type(results)
-        return results
-
-    def generate_hotwords_list(self, hotword_list_or_file):
-        # for None
-        if hotword_list_or_file is None:
-            hotword_list = None
-        # for local txt inputs
-        elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith('.txt'):
-            logging.info("Attempting to parse hotwords from local txt...")
-            hotword_list = []
-            hotword_str_list = []
-            with codecs.open(hotword_list_or_file, 'r') as fin:
-                for line in fin.readlines():
-                    hw = line.strip()
-                    hotword_str_list.append(hw)
-                    hotword_list.append(self.converter.tokens2ids([i for i in hw]))
-                hotword_list.append([self.asr_model.sos])
-                hotword_str_list.append('<s>')
-            logging.info("Initialized hotword list from file: {}, hotword list: {}."
-                         .format(hotword_list_or_file, hotword_str_list))
-        # for url, download and generate txt
-        elif hotword_list_or_file.startswith('http'):
-            logging.info("Attempting to parse hotwords from url...")
-            work_dir = tempfile.TemporaryDirectory().name
-            if not os.path.exists(work_dir):
-                os.makedirs(work_dir)
-            text_file_path = os.path.join(work_dir, os.path.basename(hotword_list_or_file))
-            local_file = requests.get(hotword_list_or_file)
-            open(text_file_path, "wb").write(local_file.content)
-            hotword_list_or_file = text_file_path
-            hotword_list = []
-            hotword_str_list = []
-            with codecs.open(hotword_list_or_file, 'r') as fin:
-                for line in fin.readlines():
-                    hw = line.strip()
-                    hotword_str_list.append(hw)
-                    hotword_list.append(self.converter.tokens2ids([i for i in hw]))
-                hotword_list.append([self.asr_model.sos])
-                hotword_str_list.append('<s>')
-            logging.info("Initialized hotword list from file: {}, hotword list: {}."
-                         .format(hotword_list_or_file, hotword_str_list))
-        # for text str input
-        elif not hotword_list_or_file.endswith('.txt'):
-            logging.info("Attempting to parse hotwords as str...")
-            hotword_list = []
-            hotword_str_list = []
-            for hw in hotword_list_or_file.strip().split():
-                hotword_str_list.append(hw)
-                hotword_list.append(self.converter.tokens2ids([i for i in hw]))
-            hotword_list.append([self.asr_model.sos])
-            hotword_str_list.append('<s>')
-            logging.info("Hotword list: {}.".format(hotword_str_list))
-        else:
-            hotword_list = None
-        return hotword_list
+from funasr.utils.vad_utils import slice_padding_fbank
+from funasr.bin.asr_inference_paraformer import Speech2Text
+# class Speech2Text:
+#     """Speech2Text class
+#
+#     Examples:
+#             >>> import soundfile
+#             >>> speech2text = Speech2Text("asr_config.yml", "asr.pb")
+#             >>> audio, rate = soundfile.read("speech.wav")
+#             >>> speech2text(audio)
+#             [(text, token, token_int, hypothesis object), ...]
+#
+#     """
+#
+#     def __init__(
+#             self,
+#             asr_train_config: Union[Path, str] = None,
+#             asr_model_file: Union[Path, str] = None,
+#             cmvn_file: Union[Path, str] = None,
+#             lm_train_config: Union[Path, str] = None,
+#             lm_file: Union[Path, str] = None,
+#             token_type: str = None,
+#             bpemodel: str = None,
+#             device: str = "cpu",
+#             maxlenratio: float = 0.0,
+#             minlenratio: float = 0.0,
+#             dtype: str = "float32",
+#             beam_size: int = 20,
+#             ctc_weight: float = 0.5,
+#             lm_weight: float = 1.0,
+#             ngram_weight: float = 0.9,
+#             penalty: float = 0.0,
+#             nbest: int = 1,
+#             frontend_conf: dict = None,
+#             hotword_list_or_file: str = None,
+#             **kwargs,
+#     ):
+#         assert check_argument_types()
+#
+#         # 1. Build ASR model
+#         scorers = {}
+#         asr_model, asr_train_args = ASRTask.build_model_from_file(
+#             asr_train_config, asr_model_file, cmvn_file=cmvn_file, device=device
+#         )
+#         frontend = None
+#         if asr_model.frontend is not None and asr_train_args.frontend_conf is not None:
+#             frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+#
+#         # logging.info("asr_model: {}".format(asr_model))
+#         # logging.info("asr_train_args: {}".format(asr_train_args))
+#         asr_model.to(dtype=getattr(torch, dtype)).eval()
+#
+#         if asr_model.ctc != None:
+#             ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
+#             scorers.update(
+#                 ctc=ctc
+#             )
+#         token_list = asr_model.token_list
+#         scorers.update(
+#             length_bonus=LengthBonus(len(token_list)),
+#         )
+#
+#         # 2. Build Language model
+#         if lm_train_config is not None:
+#             lm, lm_train_args = LMTask.build_model_from_file(
+#                 lm_train_config, lm_file, device
+#             )
+#             scorers["lm"] = lm.lm
+#
+#         # 3. Build ngram model
+#         # ngram is not supported now
+#         ngram = None
+#         scorers["ngram"] = ngram
+#
+#         # 4. Build BeamSearch object
+#         # transducer is not supported now
+#         beam_search_transducer = None
+#
+#         weights = dict(
+#             decoder=1.0 - ctc_weight,
+#             ctc=ctc_weight,
+#             lm=lm_weight,
+#             ngram=ngram_weight,
+#             length_bonus=penalty,
+#         )
+#         beam_search = BeamSearch(
+#             beam_size=beam_size,
+#             weights=weights,
+#             scorers=scorers,
+#             sos=asr_model.sos,
+#             eos=asr_model.eos,
+#             vocab_size=len(token_list),
+#             token_list=token_list,
+#             pre_beam_score_key=None if ctc_weight == 1.0 else "full",
+#         )
+#
+#         beam_search.to(device=device, dtype=getattr(torch, dtype)).eval()
+#         for scorer in scorers.values():
+#             if isinstance(scorer, torch.nn.Module):
+#                 scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
+#
+#         logging.info(f"Decoding device={device}, dtype={dtype}")
+#
+#         # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
+#         if token_type is None:
+#             token_type = asr_train_args.token_type
+#         if bpemodel is None:
+#             bpemodel = asr_train_args.bpemodel
+#
+#         if token_type is None:
+#             tokenizer = None
+#         elif token_type == "bpe":
+#             if bpemodel is not None:
+#                 tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel)
+#             else:
+#                 tokenizer = None
+#         else:
+#             tokenizer = build_tokenizer(token_type=token_type)
+#         converter = TokenIDConverter(token_list=token_list)
+#         logging.info(f"Text tokenizer: {tokenizer}")
+#
+#         self.asr_model = asr_model
+#         self.asr_train_args = asr_train_args
+#         self.converter = converter
+#         self.tokenizer = tokenizer
+#
+#         # 6. [Optional] Build hotword list from str, local file or url
+#         self.hotword_list = None
+#         self.hotword_list = self.generate_hotwords_list(hotword_list_or_file)
+#
+#         is_use_lm = lm_weight != 0.0 and lm_file is not None
+#         if (ctc_weight == 0.0 or asr_model.ctc == None) and not is_use_lm:
+#             beam_search = None
+#         self.beam_search = beam_search
+#         logging.info(f"Beam_search: {self.beam_search}")
+#         self.beam_search_transducer = beam_search_transducer
+#         self.maxlenratio = maxlenratio
+#         self.minlenratio = minlenratio
+#         self.device = device
+#         self.dtype = dtype
+#         self.nbest = nbest
+#         self.frontend = frontend
+#         self.encoder_downsampling_factor = 1
+#         if asr_train_args.encoder_conf["input_layer"] == "conv2d":
+#             self.encoder_downsampling_factor = 4
+#
+#     @torch.no_grad()
+#     def __call__(
+#             self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None,
+#             begin_time: int = 0, end_time: int = None,
+#     ):
+#         """Inference
+#
+#         Args:
+#                 speech: Input speech data
+#         Returns:
+#                 text, token, token_int, hyp
+#
+#         """
+#         assert check_argument_types()
+#
+#         # Input as audio signal
+#         if isinstance(speech, np.ndarray):
+#             speech = torch.tensor(speech)
+#
+#         if self.frontend is not None:
+#             feats, feats_len = self.frontend.forward(speech, speech_lengths)
+#             # fbanks, fbanks_len = self.frontend.forward_fbank(speech, speech_lengths)
+#             # feats, feats_len = self.frontend.forward_lfr_cmvn(speech, speech_lengths)
+#             feats = to_device(feats, device=self.device)
+#             feats_len = feats_len.int()
+#             self.asr_model.frontend = None
+#         else:
+#             feats = speech
+#             feats_len = speech_lengths
+#         lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
+#         batch = {"speech": feats, "speech_lengths": feats_len}
+#
+#         # a. To device
+#         batch = to_device(batch, device=self.device)
+#
+#         # b. Forward Encoder
+#         enc, enc_len = self.asr_model.encode(**batch)
+#         if isinstance(enc, tuple):
+#             enc = enc[0]
+#         # assert len(enc) == 1, len(enc)
+#         enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
+#
+#         predictor_outs = self.asr_model.calc_predictor(enc, enc_len)
+#         pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = predictor_outs[0], predictor_outs[1], \
+#                                                                         predictor_outs[2], predictor_outs[3]
+#         pre_token_length = pre_token_length.round().long()
+#         if torch.max(pre_token_length) < 1:
+#             return []
+#
+#         if not isinstance(self.asr_model, ContextualParaformer):
+#             if self.hotword_list:
+#                 logging.warning("Hotword is given but asr model is not a ContextualParaformer.")
+#             decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
+#             decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
+#         else:
+#             decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length, hw_list=self.hotword_list)
+#             decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
+#
+#         if isinstance(self.asr_model, BiCifParaformer):
+#             _, _, us_alphas, us_peaks = self.asr_model.calc_predictor_timestamp(enc, enc_len,
+#                                                                                    pre_token_length)  # test no bias cif2
+#
+#         results = []
+#         b, n, d = decoder_out.size()
+#         for i in range(b):
+#             x = enc[i, :enc_len[i], :]
+#             am_scores = decoder_out[i, :pre_token_length[i], :]
+#             if self.beam_search is not None:
+#                 nbest_hyps = self.beam_search(
+#                     x=x, am_scores=am_scores, maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
+#                 )
+#
+#                 nbest_hyps = nbest_hyps[: self.nbest]
+#             else:
+#                 yseq = am_scores.argmax(dim=-1)
+#                 score = am_scores.max(dim=-1)[0]
+#                 score = torch.sum(score, dim=-1)
+#                 # pad with mask tokens to ensure compatibility with sos/eos tokens
+#                 yseq = torch.tensor(
+#                     [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
+#                 )
+#                 nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
+#
+#             for hyp in nbest_hyps:
+#                 assert isinstance(hyp, (Hypothesis)), type(hyp)
+#
+#                 # remove sos/eos and get results
+#                 last_pos = -1
+#                 if isinstance(hyp.yseq, list):
+#                     token_int = hyp.yseq[1:last_pos]
+#                 else:
+#                     token_int = hyp.yseq[1:last_pos].tolist()
+#
+#                 # remove blank symbol id, which is assumed to be 0
+#                 token_int = list(filter(lambda x: x != 0 and x != 2, token_int))
+#                 if len(token_int) == 0:
+#                     continue
+#
+#                 # Change integer-ids to tokens
+#                 token = self.converter.ids2tokens(token_int)
+#
+#                 if self.tokenizer is not None:
+#                     text = self.tokenizer.tokens2text(token)
+#                 else:
+#                     text = None
+#
+#                 if isinstance(self.asr_model, BiCifParaformer):
+#                     _, timestamp = ts_prediction_lfr6_standard(us_alphas[i],
+#                                                             us_peaks[i],
+#                                                             copy.copy(token),
+#                                                             vad_offset=begin_time)
+#                     results.append((text, token, token_int, timestamp, enc_len_batch_total, lfr_factor))
+#                 else:
+#                     results.append((text, token, token_int, enc_len_batch_total, lfr_factor))
+#
+#         # assert check_return_type(results)
+#         return results
+#
+#     def generate_hotwords_list(self, hotword_list_or_file):
+#         # for None
+#         if hotword_list_or_file is None:
+#             hotword_list = None
+#         # for local txt inputs
+#         elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith('.txt'):
+#             logging.info("Attempting to parse hotwords from local txt...")
+#             hotword_list = []
+#             hotword_str_list = []
+#             with codecs.open(hotword_list_or_file, 'r') as fin:
+#                 for line in fin.readlines():
+#                     hw = line.strip()
+#                     hotword_str_list.append(hw)
+#                     hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+#                 hotword_list.append([self.asr_model.sos])
+#                 hotword_str_list.append('<s>')
+#             logging.info("Initialized hotword list from file: {}, hotword list: {}."
+#                          .format(hotword_list_or_file, hotword_str_list))
+#         # for url, download and generate txt
+#         elif hotword_list_or_file.startswith('http'):
+#             logging.info("Attempting to parse hotwords from url...")
+#             work_dir = tempfile.TemporaryDirectory().name
+#             if not os.path.exists(work_dir):
+#                 os.makedirs(work_dir)
+#             text_file_path = os.path.join(work_dir, os.path.basename(hotword_list_or_file))
+#             local_file = requests.get(hotword_list_or_file)
+#             open(text_file_path, "wb").write(local_file.content)
+#             hotword_list_or_file = text_file_path
+#             hotword_list = []
+#             hotword_str_list = []
+#             with codecs.open(hotword_list_or_file, 'r') as fin:
+#                 for line in fin.readlines():
+#                     hw = line.strip()
+#                     hotword_str_list.append(hw)
+#                     hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+#                 hotword_list.append([self.asr_model.sos])
+#                 hotword_str_list.append('<s>')
+#             logging.info("Initialized hotword list from file: {}, hotword list: {}."
+#                          .format(hotword_list_or_file, hotword_str_list))
+#         # for text str input
+#         elif not hotword_list_or_file.endswith('.txt'):
+#             logging.info("Attempting to parse hotwords as str...")
+#             hotword_list = []
+#             hotword_str_list = []
+#             for hw in hotword_list_or_file.strip().split():
+#                 hotword_str_list.append(hw)
+#                 hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+#             hotword_list.append([self.asr_model.sos])
+#             hotword_str_list.append('<s>')
+#             logging.info("Hotword list: {}.".format(hotword_str_list))
+#         else:
+#             hotword_list = None
+#         return hotword_list
 
 
 def inference(
@@ -611,15 +607,17 @@ def inference_modelscope(
             assert len(keys) == _bs, f"{len(keys)} != {_bs}"
 
             vad_results = speech2vadsegment(**batch)
-            fbanks, vadsegments = vad_results[0], vad_results[1]
+            _, vadsegments = vad_results[0], vad_results[1]
+            speech, speech_lengths = batch["speech"],  batch["speech_lengths"]
             for i, segments in enumerate(vadsegments):
                 result_segments = [["", [], [], []]]
-                for j, segment_idx in enumerate(segments):
-                    bed_idx, end_idx = int(segment_idx[0] / 10), int(segment_idx[1] / 10)
-                    segment = fbanks[:, bed_idx:end_idx, :].to(device)
-                    speech_lengths = torch.Tensor([end_idx - bed_idx]).int().to(device)
-                    batch = {"speech": segment, "speech_lengths": speech_lengths, "begin_time": vadsegments[i][j][0],
-                             "end_time": vadsegments[i][j][1]}
+                # for j, segment_idx in enumerate(segments):
+                for j, beg_idx in enumerate(range(0, len(segments), batch_size)):
+                    end_idx = min(len(segments), beg_idx + batch_size)
+                    speech_j, speech_lengths_j = slice_padding_fbank(speech, speech_lengths, segments[beg_idx:end_idx])
+
+                    batch = {"speech": speech_j, "speech_lengths": speech_lengths_j}
+                    batch = to_device(batch, device=device)
                     results = speech2text(**batch)
                     if len(results) < 1:
                         continue
@@ -633,8 +631,8 @@ def inference_modelscope(
 
                 key = keys[0]
                 result = result_segments[0]
-                text, token, token_int = result[0], result[1], result[2]
-                time_stamp = None if len(result) < 4 else result[3]
+                text, token, token_int, hyp = result[0], result[1], result[2], result[3]
+                time_stamp = None if len(result) < 5 else result[4]
 
 
                 if use_timestamp and time_stamp is not None: 

funasr/utils/vad_utils.py

@@ -0,0 +1,18 @@
+import torch
+from torch.nn.utils.rnn import pad_sequence
+
+def slice_padding_fbank(speech, speech_lengths, vad_segments):
+	speech_list = []
+	speech_lengths_list = []
+	for i, segment in enumerate(vad_segments):
+		
+		bed_idx = int(segment[0]*16)
+		end_idx = min(int(segment[1]*16), speech_lengths[0])
+		speech_i = speech[0, bed_idx: end_idx]
+		speech_lengths_i = end_idx-bed_idx
+		speech_list.append(speech_i)
+		speech_lengths_list.append(speech_lengths_i)
+	feats_pad = pad_sequence(speech_list, batch_first=True, padding_value=0.0)
+	speech_lengths_pad = torch.Tensor(speech_lengths_list).int()
+	return feats_pad, speech_lengths_pad
+