战魂小筑

讨论群:309800774 知乎关注:http://zhihu.com/people/sunicdavy 开源项目:https://github.com/davyxu

:: 首页 :: 新随笔 :: 联系 :: 聚合

:: 管理 ::

257 随笔 :: 0 文章 :: 506 评论 :: 0 Trackbacks

跨平台Unicode与UTF8互转代码

参考来源:http://blog.csdn.net/flying8127/article/details/1598521

在原来原基础上,将代码整理,并加强安全性. 并按照WindowsAPI设计, 添加输出缓冲长度探测功能

当OutUTFString为NULL时, 可以进行输出的UTF8字符串长度探测

   1:  uint32 UniCharToUTF8(wchar_t UniChar, char *OutUTFString)

   2:      {

3:

   4:          uint32 UTF8CharLength = 0;

5:

   6:          if (UniChar < 0x80)

   7:          {

   8:              if ( OutUTFString )

   9:                  OutUTFString[UTF8CharLength++] = (char)UniChar;

  10:              else

  11:                  UTF8CharLength++;

  12:          }

  13:          else if(UniChar < 0x800)

  14:          {

  15:              if ( OutUTFString )

  16:              {

  17:                  OutUTFString[UTF8CharLength++] = 0xc0 | ( UniChar >> 6 );

  18:                  OutUTFString[UTF8CharLength++] = 0x80 | ( UniChar & 0x3f );

  19:              }

  20:              else

  21:              {

  22:                  UTF8CharLength += 2;

  23:              }

  24:          }

  25:          else if(UniChar < 0x10000 )

  26:          {

  27:              if ( OutUTFString )

  28:              {

  29:                  OutUTFString[UTF8CharLength++] = 0xe0 | ( UniChar >> 12 );

  30:                  OutUTFString[UTF8CharLength++] = 0x80 | ( (UniChar >> 6) & 0x3f );

  31:                  OutUTFString[UTF8CharLength++] = 0x80 | ( UniChar & 0x3f );

  32:              }

  33:              else

  34:              {

  35:                  UTF8CharLength += 3;

  36:              }

  37:          }

  38:          else if( UniChar < 0x200000 )

  39:          {

  40:              if ( OutUTFString )

  41:              {

  42:                  OutUTFString[UTF8CharLength++] = 0xf0 | ( (int)UniChar >> 18 );

  43:                  OutUTFString[UTF8CharLength++] = 0x80 | ( (UniChar >> 12) & 0x3f );

  44:                  OutUTFString[UTF8CharLength++] = 0x80 | ( (UniChar >> 6) & 0x3f );

  45:                  OutUTFString[UTF8CharLength++] = 0x80 | ( UniChar & 0x3f );

  46:              }

  47:              else

  48:              {

  49:                  UTF8CharLength += 4;

  50:              }

51:

  52:          }

53:

  54:          return UTF8CharLength;

  55:      }

当OutUnicodeString为NULL时, 可以进行输出的Unicode字符串长度探测

   1:  uint32 UTF8StrToUnicode( const char* UTF8String, uint32 UTF8StringLength, wchar_t* OutUnicodeString, uint32 UnicodeStringBufferSize )

   2:      {

   3:          uint32 UTF8Index = 0;

   4:          uint32 UniIndex = 0;

5:

   6:          while ( UTF8Index < UTF8StringLength )

   7:          {

   8:              unsigned char UTF8Char = UTF8String[UTF8Index];

9:

  10:              if ( UnicodeStringBufferSize != 0 && UniIndex >= UnicodeStringBufferSize )

  11:                  break;

12:

  13:              if ((UTF8Char & 0x80) == 0)

  14:              {

  15:                  const uint32 cUTF8CharRequire = 1;

16:

  17:                  // UTF8字码不足

  18:                  if ( UTF8Index + cUTF8CharRequire > UTF8StringLength )

  19:                      break;

20:

  21:                  if ( OutUnicodeString )

  22:                  {

  23:                      wchar_t& WideChar = OutUnicodeString[UniIndex];

24:

  25:                      WideChar = UTF8Char;

  26:                  }

27:

  28:                  UTF8Index++;

29:

  30:              }

  31:              else if((UTF8Char & 0xE0) == 0xC0)  ///< 110x-xxxx 10xx-xxxx

  32:              {

  33:                  const uint32 cUTF8CharRequire = 2;

34:

  35:                  // UTF8字码不足

  36:                  if ( UTF8Index + cUTF8CharRequire > UTF8StringLength )

  37:                      break;

38:

  39:                  if ( OutUnicodeString )

  40:                  {

  41:                      wchar_t& WideChar = OutUnicodeString[UniIndex];

  42:                      WideChar  = (UTF8String[UTF8Index + 0] & 0x3F) << 6;

  43:                      WideChar |= (UTF8String[UTF8Index + 1] & 0x3F);

  44:                  }

45:

  46:                  UTF8Index += cUTF8CharRequire;

  47:              }

  48:              else if((UTF8Char & 0xF0) == 0xE0)  ///< 1110-xxxx 10xx-xxxx 10xx-xxxx

  49:              {

  50:                  const uint32 cUTF8CharRequire = 3;

51:

  52:                  // UTF8字码不足

  53:                  if ( UTF8Index + cUTF8CharRequire > UTF8StringLength )

  54:                      break;

55:

  56:                  if ( OutUnicodeString )

  57:                  {

  58:                      wchar_t& WideChar = OutUnicodeString[UniIndex];

59:

  60:                      WideChar  = (UTF8String[UTF8Index + 0] & 0x1F) << 12;

  61:                      WideChar |= (UTF8String[UTF8Index + 1] & 0x3F) << 6;

  62:                      WideChar |= (UTF8String[UTF8Index + 2] & 0x3F);

  63:                  }

64:

65:

  66:                  UTF8Index += cUTF8CharRequire;

  67:              }

  68:              else if((UTF8Char & 0xF8) == 0xF0)  ///< 1111-0xxx 10xx-xxxx 10xx-xxxx 10xx-xxxx

  69:              {

  70:                  const uint32 cUTF8CharRequire = 4;

71:

  72:                  // UTF8字码不足

  73:                  if ( UTF8Index + cUTF8CharRequire > UTF8StringLength )

  74:                      break;

75:

  76:                  if ( OutUnicodeString )

  77:                  {

  78:                      wchar_t& WideChar = OutUnicodeString[UniIndex];

79:

  80:                      WideChar  = (UTF8String[UTF8Index + 0] & 0x0F) << 18;

  81:                      WideChar  = (UTF8String[UTF8Index + 1] & 0x3F) << 12;

  82:                      WideChar |= (UTF8String[UTF8Index + 2] & 0x3F) << 6;

  83:                      WideChar |= (UTF8String[UTF8Index + 3] & 0x3F);

  84:                  }

85:

  86:                  UTF8Index += cUTF8CharRequire;

  87:              }

  88:              else ///< 1111-10xx 10xx-xxxx 10xx-xxxx 10xx-xxxx 10xx-xxxx

  89:              {

  90:                  const uint32 cUTF8CharRequire = 5;

91:

  92:                  // UTF8字码不足

  93:                  if ( UTF8Index + cUTF8CharRequire > UTF8StringLength )

  94:                      break;

95:

  96:                  if ( OutUnicodeString )

  97:                  {

  98:                      wchar_t& WideChar = OutUnicodeString[UniIndex];

99:

 100:                      WideChar  = (UTF8String[UTF8Index + 0] & 0x07) << 24;

 101:                      WideChar  = (UTF8String[UTF8Index + 1] & 0x3F) << 18;

 102:                      WideChar  = (UTF8String[UTF8Index + 2] & 0x3F) << 12;

 103:                      WideChar |= (UTF8String[UTF8Index + 3] & 0x3F) << 6;

 104:                      WideChar |= (UTF8String[UTF8Index + 4] & 0x3F);

 105:                  }

 106:

 107:                  UTF8Index += cUTF8CharRequire;

 108:              }

 109:

 110:

 111:              UniIndex++;

 112:          }

 113:

 114:          return UniIndex;

 115:      }

疗效: 用了此代码啊, 再也不用被iconv折磨了

posted on 2012-02-27 14:21 战魂小筑阅读(4650) 评论(9) 编辑收藏引用所属分类: 网络服务器技术、C++/ 编程语言

# re: 跨平台Unicode与UTF8互转代码 2012-02-27 21:09 我要去拯救世界

谢谢分享了！回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-02-28 17:30 天下

跨平台使用挺好,
在WIN32下,只要CW2A,CA2W宏全部搞定

回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-03-02 10:08 pillaridge

这个可能有点儿问题，wchar_t在windows下是16位，在*nix下是32位。回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-03-02 10:13 pillaridge

可以借用CLANG源代码 basic下的ConvertUTF.h和ConvertUTF.c 回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-03-09 18:04 战魂小筑

@pillaridge
感谢提醒
回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-11-19 14:22 陈成

100: WideChar = (UTF8String[UTF8Index + 0] & 0x07) << 24;
101: WideChar = (UTF8String[UTF8Index + 1] & 0x3F) << 18;
102: WideChar = (UTF8String[UTF8Index + 2] & 0x3F) << 12;
这三行不是只有最后一行才起作用吗？回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2012-11-20 10:09 战魂小筑

@陈成
这段代码是从其他地方拷贝过来的回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2013-06-07 17:03 zibbleR

UniCharToUTF8 在转换中文的时候出来的貌似还是乱码，回复更多评论

# re: 跨平台Unicode与UTF8互转代码 2013-10-15 14:57 zozo

关键是要gb2312-utf8的转换，有木有回复更多评论

刷新评论列表

只有注册用户登录后才能发表评论。


相关文章: 游戏与Web的服务器对比 Golang 热更新研究笔记 MMO相位技术研究 Golang的简单反射性能测试服务器开发语言比较开源Golang游戏服务器框架cellnet pp助手服务器端支付的RSA的奇葩公钥解密设计对golang服务器开发模式的一些思考大服务器架构讨论游戏数据库选型mysql,mongo, redis, memcached

网站导航: 博客园 IT新闻 BlogJava 知识库博问管理

战魂小筑

公告

常用链接

随笔分类(379)

随笔档案(252)

Game

最新随笔

搜索

积分与排名

最新评论

评论