FunASR/fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py

import pynini
from fun_text_processing.text_normalization.zh.graph_utils import (
    FUN_ALPHA,
    FUN_DIGIT,
    FUN_PUNCT,
    FUN_SIGMA,
    FUN_WHITE_SPACE,
    GraphFst,
)
from fun_text_processing.text_normalization.zh.utils import get_abs_path
from pynini.lib import pynutil, utf8


class PostProcessor(GraphFst):
    '''
        Postprocessing of TN, now contains:
            1. punctuation removal
            2. letter case conversion
            3. oov tagger
    '''

    def __init__(
        self, remove_puncts: bool = False, to_upper: bool = False, to_lower: bool = False, tag_oov: bool = False,
    ):
        super().__init__(name="PostProcessor", kind="processor")

        graph = pynini.cdrewrite('', '', '', FUN_SIGMA)
        if remove_puncts:
            remove_puncts_graph = pynutil.delete(
                pynini.union(FUN_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv')))
            )
            graph @= pynini.cdrewrite(remove_puncts_graph, "", "", FUN_SIGMA).optimize()

        if to_upper or to_lower:
            if to_upper:
                conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')))
            else:
                conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))

            graph @= pynini.cdrewrite(conv_cases_graph, "", "", FUN_SIGMA).optimize()

        if tag_oov:
            zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv"))
            zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv"))

            zh_charset = (
                zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv"))
            )
            en_charset = FUN_DIGIT | FUN_ALPHA | FUN_PUNCT | FUN_WHITE_SPACE
            charset = zh_charset | en_charset

            with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f:
                tags = f.readline().strip().split('\t')
                assert len(tags) == 2
                ltag, rtag = tags

            oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset)
            tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag)
            graph @= pynini.cdrewrite(tag_oov_graph, "", "", FUN_SIGMA).optimize()

        self.fst = graph.optimize()