FunASR/runtime/onnxruntime/src/encode_converter.cpp

/**
 * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
 * MIT License  (https://opensource.org/licenses/MIT)
*/
#include "encode_converter.h"
#include <assert.h>


namespace funasr {
using namespace std;

U16CHAR_T UTF16[8];
U8CHAR_T UTF8[8];

size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16);
size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8);


void EncodeConverter::SwapEndian(U16CHAR_T* pbuf, size_t len)
{
  for (size_t i = 0; i < len; i++) {
    pbuf[i] = ((pbuf[i] >> 8) | (pbuf[i] << 8));
  }
}


size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
{
  size_t n = 0;
  if (pu16[0] <= 0x007F)
  {
    pu8[0] = (pu16[0] & 0x7F);
    n = 1;
  }
  else if (pu16[0] >= 0x0080 &&  pu16[0] <= 0x07FF)
  {
    pu8[1] = (0x80 | (pu16[0] & 0x003F));
    pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
    n = 2;
  }
  else if (pu16[0] >= 0x0800)
  {
    pu8[2] = (0x80 | (pu16[0] & 0x003F));
    pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
    pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
    n = 3;
  }

  return n;
}

#define is2ByteUtf16(u16) ( (u16) >= 0x0080 && (u16) <= 0x07FF )
#define is3ByteUtf16(u16) ( (u16) >= 0x0800 )

size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
{
  size_t n = 0;
  if (pu16[0] <= 0x007F)
  {
    pu8[0] = (pu16[0] & 0x7F);
    n = 1;
  }
  else if (pu16[0] >= 0x0080 &&  pu16[0] <= 0x07FF)
  {
    pu8[1] = (0x80 | (pu16[0] & 0x003F));
    pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
    n = 2;
  }
  else if (pu16[0] >= 0x0800)
  {
    pu8[2] = (0x80 | (pu16[0] & 0x003F));
    pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
    pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
    n = 3;
  }

  return n;
}

size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, size_t ilen,
    U8CHAR_T* pu8, size_t olen)
{
  size_t offset = 0;
  size_t sz = 0;
  /*
  for (size_t i = 0; i < ilen && offset < static_cast<int>(olen) - 3; i++) {
    sz = utf16ToUtf8(pu16 + i, pu8 + offset);
    offset += sz;
  }
  */
  for (size_t i = 0; i < ilen && static_cast<int>(offset) < static_cast<int>(olen); i++) {
    sz = Utf16ToUtf8(pu16 + i, pu8 + offset);
    if (static_cast<int>(offset + static_cast<int>(sz)) <= static_cast<int>(olen))
        offset += sz;
  }
  
 // pu8[offset] = '\0';
  return offset;
}

u8string EncodeConverter::Utf16ToUtf8(const u16string& u16str)
{
  size_t buflen = u16str.length()*3 + 1;
  U8CHAR_T* pu8 = new U8CHAR_T[buflen];
  size_t len = Utf16ToUtf8(u16str.data(), u16str.length(),
    pu8, buflen);
  u8string u8str(pu8, len);
  delete [] pu8;

  return u8str;
}

size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, U16CHAR_T* pu16)
{
  size_t n = 0;
  if ((pu8[0] & 0xF0) == 0xE0)
  {
    if ((pu8[1] & 0xC0) == 0x80 &&
        (pu8[2] & 0xC0) == 0x80)
    {
      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
    }
    else
    {
      pu16[0] = defUniChar;
    }
    n = 3;
  } 
  else if ((pu8[0] & 0xE0) == 0xC0)
  {
    if ((pu8[1] & 0xC0) == 0x80) 
    {
      pu16[0] = ((pu8[0] & 0x1C) >> 2);
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
    }
    else
    {
      pu16[0] = defUniChar;
    }
    n = 2;
  } 
  else if ((pu8[0] & 0x80) == 0x00) 
  {
    pu16[0] = pu8[0];
    n = 1;
  }

  return n;
}

size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
{
  size_t n = 0;
  if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
  {
    if ((pu8[1] & 0xC0) == 0x80 &&
        (pu8[2] & 0xC0) == 0x80)
    {
      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
      n = 3;
    }
    else
    {
      pu16[0] = 0x0000;
      n = 1;
    }
  } 
  else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
  {
    if ((pu8[1] & 0xC0) == 0x80) 
    {
      pu16[0] = ((pu8[0] & 0x1C) >> 2);
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
      n = 2;
    }
    else
    {
      pu16[0] = 0x0000;
      n = 1;
    }
  } 
  else if ((pu8[0] & 0x80) == 0x00) 
  {
    pu16[0] = pu8[0];
    n = 1;
  }
  else
  {
      pu16[0] = 0x0000;
      n = 1;
  }
  return n;
}

size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
{
  size_t n = 0;
  if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
  {
    if ((pu8[1] & 0xC0) == 0x80 &&
        (pu8[2] & 0xC0) == 0x80)
    {
      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
      n = 3;
      if( !is3ByteUtf16(pu16[0]) )
      {
          pu16[0] = 0x0000;
          n = 1;
      }
    }
    else
    {
      pu16[0] = 0x0000;
      n = 1;
    }
  } 
  else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
  {
    if ((pu8[1] & 0xC0) == 0x80) 
    {
      pu16[0] = ((pu8[0] & 0x1C) >> 2);
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
      n = 2;
      if( !is2ByteUtf16(pu16[0]) )
      {
          pu16[0] = 0x0000;
          n = 1;
      }
    }
    else
    {
      pu16[0] = 0x0000;
      n = 1;
    }
  } 
  else if ((pu8[0] & 0x80) == 0x00) 
  {
    pu16[0] = pu8[0];
    n = 1;
  }
  else
  {
      pu16[0] = 0x0000;
      n = 1;
  }

  return n;
  /*
  size_t n = 0;
  if ((pu8[0] & 0xF0) == 0xE0)
  {
    if (ilen >= 3 && (pu8[1] & 0xC0) == 0x80 &&
        (pu8[2] & 0xC0) == 0x80)
    {
      pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
    }
    else
    {
      pu16[0] = defUniChar;
    }
    n = 3;
  } 
  else if ((pu8[0] & 0xE0) == 0xC0)
  {
    if( ilen >= 2 && (pu8[1] & 0xC0) == 0x80) 
    {
      pu16[0] = ((pu8[0] & 0x1C) >> 2);
      pu16[0] <<= 8;
      pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
    }
    else
    {
      pu16[0] = defUniChar;
    }
    n = 2;
  } 
  else if ((pu8[0] & 0x80) == 0x00) 
  {
    pu16[0] = pu8[0];
    n = 1;
  }
  else
  {
      pu16[0] = defUniChar;
      n = 1;
      for (size_t i = 1; i < ilen; i++)
      {
          if ((pu8[i] & 0xF0) == 0xE0 || (pu8[i] & 0xE0) == 0xC0 || (pu8[i] & 0x80) == 0x00)
              break;
          n++;
      }
  }

  return n;
  */
}

size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen,
    U16CHAR_T* pu16, size_t olen)
{
  int offset = 0;
  size_t sz = 0;
  for (size_t i = 0; i < ilen && offset < static_cast<int>(olen); offset ++)
  {
    sz = Utf8ToUtf16(pu8 + i, ilen - i, pu16 + offset);
    i += sz;
    if (sz == 0) {
      // failed
      // assert(sz != 0);
      break;
    }
  }
//  pu16[offset] = '\0';

  return offset;
}

u16string EncodeConverter::Utf8ToUtf16(const u8string& u8str)
{
  U16CHAR_T* p16 = new U16CHAR_T[u8str.length() + 1];
  size_t len = Utf8ToUtf16(u8str.data(), u8str.length(), 
      p16, u8str.length() + 1);
  u16string u16str(p16, len);
  delete[] p16;

  return u16str;
}

bool EncodeConverter::IsUTF8(const U8CHAR_T* pu8, size_t ilen)
{
  size_t i;
  size_t n = 0;
  for (i = 0; i < ilen; i += n)
  {
    if ((pu8[i] & 0xF0) == 0xE0 &&
        (pu8[i + 1] & 0xC0) == 0x80 &&
        (pu8[i + 2] & 0xC0) == 0x80)
    {
      n = 3;
    }
    else if ((pu8[i] & 0xE0) == 0xC0 &&
        (pu8[i + 1] & 0xC0) == 0x80)
    {
      n = 2;
    }
    else if ((pu8[i] & 0x80) == 0x00)
    {
      n = 1;
    }
    else
    {
      break;
    }
  }

  return i == ilen;
}

bool EncodeConverter::IsUTF8(const u8string& u8str)
{
  return IsUTF8(u8str.data(), u8str.length());
}
  
size_t EncodeConverter::GetUTF8Len(const U8CHAR_T* pu8, size_t ilen)
{
  size_t i;
  size_t n = 0;
  size_t rlen = 0;
  for (i = 0; i < ilen; i += n, rlen ++)
  {
    if ((pu8[i] & 0xF0) == 0xE0 &&
        (pu8[i + 1] & 0xC0) == 0x80 &&
        (pu8[i + 2] & 0xC0) == 0x80)
    {
      n = 3;
    }
    else if ((pu8[i] & 0xE0) == 0xC0 &&
        (pu8[i + 1] & 0xC0) == 0x80)
    {
      n = 2;
    }
    else if ((pu8[i] & 0x80) == 0x00)
    {
      n = 1;
    }
    else
    {
      break;
    }
  }

  if (i == ilen)
    return 0;
  else
    return rlen;
}

size_t EncodeConverter::GetUTF8Len(const u8string& u8str)
{
  return GetUTF8Len(u8str.data(), u8str.length());
}


size_t EncodeConverter::Utf16ToUtf8Len(const U16CHAR_T* pu16, size_t ilen)
{
  int offset = 0;
  for (size_t i = 0; i < ilen ; i++) {
      if (pu16[i] <= 0x007F)
      {
        offset += 1;
      }
      else if (pu16[i] >= 0x0080 &&  pu16[i] <= 0x07FF)
      {
        offset += 2;
      }
      else if (pu16[i] >= 0x0800)
      {
        offset += 3;
      }
  }
  
  return offset;
}

uint16_t EncodeConverter::ToUni(const char* sc, int &len)
{
    uint16_t wide[2];
    len = (int)Utf8ToUtf16((const U8CHAR_T*)sc, wide);
    return wide[0];
}

bool EncodeConverter::IsAllChineseCharactor(const U8CHAR_T* pu8, size_t ilen) {
    if (pu8 == nullptr || ilen <= 0) {
        return false;
    }

    U16CHAR_T* p16 = new U16CHAR_T[ilen + 1];
    size_t len = Utf8ToUtf16(pu8, ilen, p16, ilen + 1);
    for (size_t i = 0; i < len; i++) {
        if (p16[i] < 0x4e00 || p16[i] > 0x9fff) {
            delete[] p16;
            return false;
        }
    }
    delete[] p16;
    return true;
}

bool EncodeConverter::HasAlpha(const U8CHAR_T* pu8, size_t ilen) {
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  for (size_t i = 0; i < ilen; i++) {
    if (pu8[i]> 0 && isalpha(pu8[i])){
      return true;
    }
  }
  return false;
}


bool EncodeConverter::IsAllAlpha(const U8CHAR_T* pu8, size_t ilen) {
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  for (size_t i = 0; i < ilen; i++) {
    if (!(pu8[i]> 0 && isalpha(pu8[i]))){
      return false;
    }
  }
  return true;
}

bool EncodeConverter::IsAllAlphaAndPunct(const U8CHAR_T* pu8, size_t ilen) {
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  bool flag1 = HasAlpha(pu8, ilen);
  if (flag1 == false) {
    return false;
  }

  for (size_t i = 0; i < ilen; i++) {
    if (!(pu8[i]> 0 && (isalpha(pu8[i]) || (ispunct(pu8[i]))))){
      return false;
    }
  }
  return true;
}

bool EncodeConverter::IsAllAlphaAndDigit(const U8CHAR_T* pu8, size_t ilen) {
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  bool flag1 = HasAlpha(pu8, ilen);
  if (flag1 == false) {
    return false;
  }

  for (size_t i = 0; i < ilen; i++) {
    if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || pu8[i] == '\''))){
      return false;
    }
  }
  return true;
}
bool EncodeConverter::IsAllAlphaAndDigitAndBlank(const U8CHAR_T* pu8, size_t ilen) {
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  for (size_t i = 0; i < ilen; i++) {
    if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || isblank(pu8[i]) || pu8[i] == '\''))){
      return false;
    }
  }
  return true;
}
bool EncodeConverter::NeedAddTailBlank(std::string str) {
  U8CHAR_T *pu8 = (U8CHAR_T*)str.data();
  size_t ilen = str.size();
  if (pu8 == nullptr || ilen <= 0) {
    return false;
  }
  if (IsAllAlpha(pu8, ilen) || IsAllAlphaAndPunct(pu8, ilen) || IsAllAlphaAndDigit(pu8, ilen)) {
    return true;
  } else {
    return false;
  }
}
std::vector<std::string> EncodeConverter::MergeEnglishWord(std::vector<std::string> &str_vec_input,
                                                           std::vector<int> &merge_mask) {
  std::vector<std::string> output;
  for (int i = 0; i < merge_mask.size(); i++) {
    if (merge_mask[i] == 1 && i > 0) {
      output[output.size() - 1] += str_vec_input[i];
    } else {
      output.push_back(str_vec_input[i]);
    }
  }
  str_vec_input.swap(output);
  return str_vec_input;
}
size_t EncodeConverter::Utf8ToCharset(const std::string &input, std::vector<std::string> &output) {
  std::string ch; 
  for (size_t i = 0, len = 0; i != input.length(); i += len) {
    unsigned char byte = (unsigned)input[i];
    if (byte >= 0xFC) // lenght 6
      len = 6;  
    else if (byte >= 0xF8)
      len = 5;
    else if (byte >= 0xF0)
      len = 4;
    else if (byte >= 0xE0)
      len = 3;
    else if (byte >= 0xC0)
      len = 2;
    else
      len = 1;
    ch = input.substr(i, len);
    output.push_back(ch);
  }   
  return output.size();
}
}