update

funasr/models/encoder/data2vec_encoder.py

@@ -12,6 +12,7 @@ import torch.nn as nn
 import torch.nn.functional as F
 from typeguard import check_argument_types
 
+from funasr.models.encoder.abs_encoder import AbsEncoder
 from funasr.modules.data2vec.data_utils import compute_mask_indices
 from funasr.modules.data2vec.ema_module import EMAModule
 from funasr.modules.data2vec.grad_multiply import GradMultiply
@@ -28,7 +29,7 @@ def get_annealed_rate(start, end, curr_step, total_steps):
     return end - r * pct_remaining
 
 
-class Data2VecEncoder(torch.nn.Module):
+class Data2VecEncoder(AbsEncoder):
     def __init__(
             self,
             # for ConvFeatureExtractionModel
@@ -573,4 +574,4 @@ class Data2VecEncoder(torch.nn.Module):
             )
 
     def output_size(self) -> int:
-        return self.encoder_embed_dim
+        return self.encoder_embed_dim

funasr/models/encoder/resnet34_encoder.py

@@ -1,8 +1,8 @@
 import torch
 from torch.nn import functional as F
+from funasr.models.encoder.abs_encoder import AbsEncoder
 from typing import Tuple, Optional
 from funasr.models.pooling.statistic_pooling import statistic_pooling, windowed_statistic_pooling
-from funasr.models.encoder.abs_encoder import AbsEncoder
 from collections import OrderedDict
 import logging
 import numpy as np
@@ -76,7 +76,7 @@ class BasicBlock(torch.nn.Module):
         return xs_pad, ilens
 
 
-class ResNet34(torch.nn.Module):
+class ResNet34(AbsEncoder):
     def __init__(
             self,
             input_size,
@@ -406,6 +406,12 @@ class ResNet34Diar(ResNet34):
             tf2torch_tensor_name_prefix_torch="encoder",
             tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder"
     ):
+        """
+        Author: Speech Lab, Alibaba Group, China
+        SOND: Speaker Overlap-aware Neural Diarization for Multi-party Meeting Analysis
+        https://arxiv.org/abs/2211.10243
+        """
+
         super(ResNet34Diar, self).__init__(
             input_size,
             use_head_conv=use_head_conv,
@@ -633,6 +639,12 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):
             tf2torch_tensor_name_prefix_torch="encoder",
             tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder"
     ):
+        """
+        Author: Speech Lab, Alibaba Group, China
+        TOLD: A Novel Two-Stage Overlap-Aware Framework for Speaker Diarization
+        https://arxiv.org/abs/2303.05397
+        """
+
         super(ResNet34SpL2RegDiar, self).__init__(
             input_size,
             use_head_conv=use_head_conv,
@@ -838,4 +850,4 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):
                 else:
                     logging.warning("{} is missed from tf checkpoint".format(name))
 
-        return var_dict_torch_update
+        return var_dict_torch_update

funasr/models/encoder/rnn_encoder.py

@@ -1,3 +1,4 @@
+
 from typing import Optional
 from typing import Sequence
 from typing import Tuple
@@ -9,11 +10,11 @@ from typeguard import check_argument_types
 from funasr.modules.nets_utils import make_pad_mask
 from funasr.modules.rnn.encoders import RNN
 from funasr.modules.rnn.encoders import RNNP
+from funasr.models.encoder.abs_encoder import AbsEncoder
 
 
-class RNNEncoder(torch.nn.Module):
+class RNNEncoder(AbsEncoder):
     """RNNEncoder class.
-
     Args:
         input_size: The number of expected features in the input
         output_size: The number of output features
@@ -22,7 +23,6 @@ class RNNEncoder(torch.nn.Module):
         use_projection: Use projection layer or not
         num_layers: Number of recurrent layers
         dropout: dropout probability
-
     """
 
     def __init__(