| 12345678910111213141516171819202122232425 |
- #!/bin/bash
- ## Make sure that srilm is installed
- dir=lm
- mkdir -p $dir
- [ -f path.sh ] && . ./path.sh
- # Prepare data, the format of the text should be:
- # BAC009S0002W0122 而 对 楼市 成交 抑制 作用 最 大 的 限 购
- # BAC009S0002W0123 也 成为 地方 政府 的 眼中 钉
- corpus=lm/text
- # generate lm dict
- cat $corpus | awk '{for(n=2;n<=NF;n++) print tolower($n); }' | \
- cat - <(echo "<unk>";echo "<s>"; echo "</s>") | \
- sort | uniq -c | sort -nr | awk '{print $2}' > $dir/corpus.dict || exit 1;
- # train ngram
- cat $corpus | awk '{for(n=2;n<=NF;n++){ printf tolower($n); if(n<NF) printf " "; else print ""; }}' > $dir/train
- ngram-count -text $dir/train -order 4 -limit-vocab -vocab $dir/corpus.dict -unk \
- -kndiscount -interpolate -gt1min 1 -gt2min 1 -gt3min 2 -gt4min 2 -lm $dir/lm.arpa
|
