add function IsPunc

funasr/runtime/onnxruntime/src/tokenizer.cpp

@@ -142,6 +142,14 @@ vector<int> CTokenizer::Punc2Ids(vector<string> input)
 	return result;
 }
 
+bool CTokenizer::IsPunc(string& Punc)
+{
+	if (m_punc2id.find(Punc) != m_punc2id.end())
+		return true;
+	else
+		return false;
+}
+
 vector<string> CTokenizer::SplitChineseString(const string & str_info)
 {
 	vector<string> list;

funasr/runtime/onnxruntime/src/tokenizer.h

@@ -30,7 +30,7 @@ public:
 	vector<string> SplitChineseString(const string& str_info);
 	void StrSplit(const string& str, const char split, vector<string>& res);
 	void Tokenize(const char* str_info, vector<string>& str_out, vector<int>& id_out);
-
+	bool IsPunc(string& Punc);
 };
 
 } // namespace funasr