| 1234567891011121314151617181920212223242526272829303132333435 |
- #!/usr/bin/env python3
- # encoding: utf-8
- import sys
- # sys.argv[1]: lm dict
- # sys.argv[2]: lexicon file
- # sys.argv[3]: lexicon file for corpus.dict
- lex_dict = {}
- with open(sys.argv[2], 'r', encoding='utf8') as fin:
- for line in fin:
- words = line.strip().split('\t')
- if len(words) != 2:
- continue
- lex_dict[words[0]] = words[1]
- with open(sys.argv[1], 'r', encoding='utf8') as fin, \
- open(sys.argv[3], 'w', encoding='utf8') as fout:
- for line in fin:
- word = line.strip()
- if word == '<s>' or word == '</s>':
- continue
- word_lex = ""
- if word in lex_dict:
- word_lex = lex_dict[word]
- else:
- for i in range(len(word)):
- if word[i] in lex_dict:
- word_lex += " " + lex_dict[word[i]]
- else:
- word_lex += " <unk>"
-
- fout.write('{}\t{}\n'.format(word, word_lex.strip()))
|
