/* * * Author wrt(guangguang) * Date 2011-06-08 * Copyright Hangzhou, Netease Inc. * Brief Utilities for string operation * */ #include "string_util.h" #include /* for vsnprintf */ #include /* for assert */ #include /* for mem*** */ #include /* va_list, va_start, va_end */ #include #include "base/macros.h" #include "base/third_party/convert_utf/ConvertUTF.h" namespace nbase { namespace { template int StringTokenizeT(const std::basic_string &input, const std::basic_string &delimitor, std::list > &output) { size_t token_begin; size_t token_end; output.clear(); for (token_begin = token_end = 0; token_end != std::basic_string::npos;) { token_begin = input.find_first_not_of(delimitor, token_begin); if (token_begin == std::basic_string::npos) break; token_end = input.find_first_of(delimitor, token_begin + 1); output.push_back(input.substr(token_begin, token_end - token_begin)); token_begin = token_end + 1; } return static_cast(output.size()); } template size_t StringReplaceAllT(const std::basic_string &find, const std::basic_string &replace, std::basic_string &output) { size_t find_length = find.size(); size_t replace_length = replace.size(); size_t offset = 0, endpos; size_t target = 0, found_pos; size_t replaced = 0; CharType *data_ptr; if (find.empty() || output.empty()) return 0; /* * to avoid extra memory reallocating, * we use two passes to finish the task in the case that replace.size() is greater find.size() */ if (find_length < replace_length) { /* the first pass, count all available 'find' to be replaced */ for (;;) { offset = output.find(find, offset); if (offset == std::basic_string::npos) break; replaced++; offset += find_length; } if (replaced == 0) return 0; size_t newsize = output.size() + replaced * (replace_length - find_length); /* we apply for more memory to hold the content to be replaced */ endpos = newsize; offset = newsize - output.size(); output.resize(newsize); data_ptr = &output[0]; memmove((void*)(data_ptr + offset), (void*)data_ptr, (output.size() - offset) * sizeof(CharType)); } else { endpos = output.size(); offset = 0; data_ptr = const_cast(&output[0]); } /* the second pass, the replacement */ while (offset < endpos) { found_pos = output.find(find, offset); if (found_pos != std::basic_string::npos) { /* move the content between two targets */ if (target != found_pos) memmove((void*)(data_ptr + target), (void*)(data_ptr + offset), (found_pos - offset) * sizeof(CharType)); target += found_pos - offset; /* replace */ memcpy(data_ptr + target, replace.data(), replace_length * sizeof(CharType)); target += replace_length; offset = find_length + found_pos; replaced++; } else { /* ending work */ if (target != offset) memcpy((void*)(data_ptr + target), (void*)(data_ptr + offset), (endpos - offset) * sizeof(CharType)); break; } } if (replace_length < find_length) output.resize(output.size() - replaced * (find_length - replace_length)); return replaced; } inline int vsnprintfT(char *dst, size_t count, const char *format, va_list ap) { return vsnprintf(dst, count, format, ap); } inline int vsnprintfT(wchar_t *dst, size_t count, const wchar_t *format, va_list ap) { #if defined(OS_WIN) return _vsnwprintf(dst, count, format, ap); #else return vswprintf(dst, count, format, ap); #endif } template void StringAppendVT(const CharType *format, va_list ap, std::basic_string &output) { CharType stack_buffer[1024]; /* first, we try to finish the task using a fixed-size buffer in the stack */ va_list ap_copy; GG_VA_COPY(ap_copy, ap); int result = vsnprintfT(stack_buffer, COUNT_OF(stack_buffer), format, ap_copy); va_end(ap_copy); if (result >= 0 && result < static_cast(COUNT_OF(stack_buffer))) { /* It fits */ output.append(stack_buffer, result); return; } /* then, we have to repeatedly increase buffer size until it fits. */ int buffer_size = COUNT_OF(stack_buffer); std::basic_string heap_buffer; for (;;) { if (result != -1) { assert(0); return; /* not expected, result should be -1 here */ } buffer_size <<= 1; /* try doubling the buffer size */ if (buffer_size > 32 * 1024 * 1024) { assert(0); return; /* too long */ } /* resize */ heap_buffer.resize(buffer_size); /* * NOTE: You can only use a va_list once. Since we're in a while loop, we * need to make a new copy each time so we don't use up the original. */ GG_VA_COPY(ap_copy, ap); result = vsnprintfT(&heap_buffer[0], buffer_size, format, ap_copy); va_end(ap_copy); if ((result >= 0) && (result < buffer_size)) { /* It fits */ output.append(&heap_buffer[0], result); return; } } } #define NOT_SPACE(x) ((x) != 0x20 && ((x) < 0 || 0x1d < (x))) template void StringTrimT(std::basic_string &output) { if (output.empty()) return; size_t bound1 = 0; size_t bound2 = output.length(); const CharType *src = output.data(); for (; bound2 > 0; bound2--) if (NOT_SPACE(src[bound2-1])) break; for (; bound1 < bound2; bound1++) if (NOT_SPACE(src[bound1])) break; if (bound1 < bound2) { memmove((void *)src, src + bound1, sizeof(CharType) * (bound2 - bound1)); } output.resize(bound2 - bound1); } template void StringTrimLeftT(std::basic_string &output) { size_t check = 0; size_t length = output.length(); const CharType *src = output.data(); for (; check < length; check++) if (NOT_SPACE(src[check])) break; output.erase(0, check); } template void StringTrimRightT(std::basic_string &output) { size_t length = output.length(); const CharType *src = output.data(); for (; length > 0; length--) if (NOT_SPACE(src[length-1])) break; output.resize(length); } } // anonymous namespace std::string StringPrintf(const char *format, ...) { va_list args; va_start(args, format); std::string output; StringAppendV(output, format, args); va_end(args); return output; } std::wstring StringPrintf(const wchar_t *format, ...) { va_list args; va_start(args, format); std::wstring output; StringAppendV(output, format, args); va_end(args); return output; } const std::string& StringPrintf(std::string &output, const char *format, ...) { va_list args; va_start(args, format); output.clear(); StringAppendV(output, format, args); va_end(args); return output; } const std::wstring& StringPrintf(std::wstring &output, const wchar_t *format, ...) { va_list args; va_start(args, format); output.clear(); StringAppendV(output, format, args); va_end(args); return output; } void StringPrintfV(std::string &output, const char *format, va_list ap) { output.clear(); StringAppendV(output, format, ap); } void StringPrintfV(std::wstring &output, const wchar_t *format, va_list ap) { output.clear(); StringAppendV(output, format, ap); } void StringAppendF(std::string &output, const char *format, ...) { va_list args; va_start(args, format); StringAppendV(output, format, args); va_end(args); } void StringAppendF(std::wstring &output, const wchar_t *format, ...) { va_list args; va_start(args, format); StringAppendV(output, format, args); va_end(args); } void StringAppendV(std::string &output, const char *format, va_list ap) { StringAppendVT(format, ap, output); } void StringAppendV(std::wstring &output, const wchar_t *format, va_list ap) { StringAppendVT(format, ap, output); } std::list StringTokenize(const char *input, const char *delimitor) { std::list output; std::string input2(input); if (input2.empty()) return output; char *token = strtok(&input2[0], delimitor); while (token != NULL) { output.push_back(token); token = strtok(NULL, delimitor); } return output; } std::list StringTokenize(const wchar_t *input, const wchar_t *delimitor) { std::list output; std::wstring input2(input); if (input2.empty()) return output; #if defined(OS_WIN) wchar_t *token = wcstok(&input2[0], delimitor); while (token != NULL) { output.push_back(token); token = wcstok(NULL, delimitor); } #else wchar_t *ptr; wchar_t *token = wcstok(const_cast(&input2[0]), delimitor, &ptr); while (token != NULL) { output.push_back(token); token = wcstok(NULL, delimitor, &ptr); } #endif return output; } int StringTokenize(const std::string& input, const std::string& delimitor, std::list& output) { return StringTokenizeT(input, delimitor, output); } int StringTokenize(const std::wstring& input, const std::wstring& delimitor, std::list& output) { return StringTokenizeT(input, delimitor, output); } size_t StringReplaceAll(const std::string& find, const std::string& replace, std::string& output) { return StringReplaceAllT(find, replace, output); } size_t StringReplaceAll(const std::wstring& find, const std::wstring& replace, std::wstring& output) { return StringReplaceAllT(find, replace, output); } void LowerString(std::string &str) { if (str.empty()) return; char *start = &str[0]; char *end = start + str.length(); for (; start < end; start++) { if (*start >= 'A' && *start <= 'Z') *start += 'a' - 'A'; } } void LowerString(std::wstring &str) { if (str.empty()) return; wchar_t *start = &str[0]; wchar_t *end = start + str.length(); for (; start < end; start++) { if (*start >= L'A' && *start <= L'Z') *start += L'a' - L'A'; } } void UpperString(std::string &str) { if (str.empty()) return; char *start = &str[0]; char *end = start + str.length(); for (; start < end; start++) { if (*start >= 'a' && *start <= 'z') *start -= 'a' - 'A'; } } void UpperString(std::wstring &str) { if (str.empty()) return; wchar_t *start = &str[0]; wchar_t *end = start + str.length(); for (; start < end; start++) { if (*start >= L'a' && *start <= L'z') *start -= L'a' - L'A'; } } std::string MakeLowerString(const std::string &src) { std::string dest(src); LowerString(dest); return dest; } std::wstring MakeLowerString(const std::wstring &src) { std::wstring dest(src); LowerString(dest); return dest; } std::string MakeUpperString(const std::string &src) { std::string dest(src); UpperString(dest); return dest; } std::wstring MakeUpperString(const std::wstring &src) { std::wstring dest(src); UpperString(dest); return dest; } void BinaryToHexString(const void *binary, size_t length, std::string &output) { static const unsigned char kHexChars[] = "0123456789abcdef"; output.resize(length << 1); if (length == 0) return; unsigned char *dst = reinterpret_cast(&output[0]); const unsigned char *src = reinterpret_cast(binary); for (size_t i = 0; i < length; i++) { dst[i<<1] = kHexChars[src[i]>>4]; dst[(i<<1)+1] = kHexChars[src[i]&0xF]; } } std::string BinaryToHexString(const void *binary, size_t length) { std::string output; BinaryToHexString(binary, length, output); return output; } std::string BinaryToHexString(const std::string &binary) { std::string output; BinaryToHexString(binary.c_str(), binary.size(), output); return output; } int HexStringToBinary(const char *input, size_t length, std::string &output) { output.resize(length >> 1); if (length == 0) return 0; const char *src = input; char *dst = &output[0]; size_t i, size = output.size(); for (i = 0; i < size; i++) { char h = HexCharToInt8(src[i<<1]); char l = HexCharToInt8(src[(i<<1)+1]); if (l == -1 || h == -1) break; dst[i] = (h<<4)|l; } // i may be shorter than size when an error occurs output.resize(i); return (int)i; } std::string HexStringToBinary(const std::string &input) { std::string output; HexStringToBinary(input.c_str(), input.size(), output); return output; } char HexCharToInt8(char c) { if (c >= '0' && c <= '9') return c - '0'; else if (c >= 'a' && c <= 'f') return c - 'a' + 10; else if (c >= 'A' && c <= 'F') return c - 'A' + 10; assert(0); // not a hexadecimal char return -1; } std::wstring UTF8ToUTF16(const UTF8Char *utf8, size_t length) { UTF16Char output[4096]; const UTF8 *src_begin = reinterpret_cast(utf8); const UTF8 *src_end = src_begin + length; UTF16 *dst_begin = reinterpret_cast(output); std::wstring utf16; while (src_begin < src_end) { ConversionResult result = ConvertUTF8toUTF16(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf16.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf16.clear(); break; } } return utf16; } std::string UTF16ToUTF8(const UTF16Char *utf16, size_t length) { UTF8Char output[8192]; const UTF16 *src_begin = reinterpret_cast(utf16); const UTF16 *src_end = src_begin + length; UTF8 *dst_begin = reinterpret_cast(output); std::string utf8; while (src_begin < src_end) { ConversionResult result = ConvertUTF16toUTF8(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf8.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf8.clear(); break; } } return utf8; } std::basic_string UTF8ToUTF32(const UTF8Char *utf8, size_t length) { UTF32Char output[4096]; const UTF8 *src_begin = reinterpret_cast(utf8); const UTF8 *src_end = src_begin + length; UTF32 *dst_begin = reinterpret_cast(output); std::basic_string utf32; while (src_begin < src_end) { ConversionResult result = ConvertUTF8toUTF32(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf32.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf32.clear(); break; } } return utf32; } std::string UTF32ToUTF8(const UTF32Char *utf32, size_t length) { UTF8Char output[8192]; const UTF32 *src_begin = reinterpret_cast(utf32); const UTF32 *src_end = src_begin + length; UTF8 *dst_begin = reinterpret_cast(output); std::string utf8; while (src_begin < src_end) { ConversionResult result = ConvertUTF32toUTF8(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf8.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf8.clear(); break; } } return utf8; } std::basic_string UTF16ToUTF32(const UTF16Char *utf16, size_t length) { UTF32Char output[4096]; const UTF16 *src_begin = reinterpret_cast(utf16); const UTF16 *src_end = src_begin + length; UTF32 *dst_begin = reinterpret_cast(output); std::basic_string utf32; while (src_begin < src_end) { ConversionResult result = ConvertUTF16toUTF32(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf32.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf32.clear(); break; } } return utf32; } std::wstring UTF32ToUTF16(const UTF32Char *utf32, size_t length) { UTF16Char output[8192]; const UTF32 *src_begin = reinterpret_cast(utf32); const UTF32 *src_end = src_begin + length; UTF16 *dst_begin = reinterpret_cast(output); std::wstring utf16; while (src_begin < src_end) { ConversionResult result = ConvertUTF32toUTF16(&src_begin, src_end, &dst_begin, dst_begin + COUNT_OF(output), lenientConversion); utf16.append(output, dst_begin - reinterpret_cast(output)); dst_begin = reinterpret_cast(output); if (result == sourceIllegal || result == sourceExhausted) { utf16.clear(); break; } } return utf16; } std::wstring UTF8ToUTF16(const std::string &utf8) { return UTF8ToUTF16(utf8.c_str(), utf8.length()); } std::string UTF16ToUTF8(const std::wstring &utf16) { return UTF16ToUTF8(utf16.c_str(), utf16.length()); } std::basic_string UTF8ToUTF32(const std::string &utf8) { return UTF8ToUTF32(utf8.c_str(), utf8.length()); } std::string UTF32ToUTF8(const std::basic_string &utf32) { return UTF32ToUTF8(utf32.c_str(), utf32.length()); } std::basic_string UTF16ToUTF32(const std::wstring &utf16) { return UTF16ToUTF32(utf16.c_str(), utf16.length()); } std::wstring UTF32ToUTF16(const std::basic_string &utf32) { return UTF32ToUTF16(utf32.c_str(), utf32.length()); } void UTF8CreateLengthTable(unsigned table[256]) { int c; for (c = 0; c < 256; c++) { if (c < 0x80) table[c] = 1; else if (0xBF < c && c < 0xE0) table[c] = 2; else if (0xDF < c && c < 0xF0) table[c] = 3; else if (0xEF < c && c < 0xF8) table[c] = 4; else table[c] = 0; } } bool ValidateUTF8Stream(const void* stream, unsigned length) { /* * 根据RFC3629(http://tools.ietf.org/html/rfc3629), * UTF-8流一个字符的第一个字节由0-4个数值为1的二进制位 * 这些位之后的第一位必须是0,当这些位的个数为0的时候表 * 示这个字符是ASCII,占用一个字节,除此之外表示这个字符 * 所占用的字节数;第二个字节开始,每个字节必须使用10的 * 二进制位开头,这样利于快速定位一个字符的起始字节,例如: * * 0XXXXXXX * 110XXXXX 10XXXXXX * 1110XXXX 10XXXXXX 10XXXXXX * 11110XXX 10XXXXXX 10XXXXXX 10XXXXXX * * 另,UTF-8理论上支持6字节长度,但是标准将其限定为4字节 */ unsigned i, j, k; unsigned char* s = (unsigned char*)stream; static unsigned int table_created = 0; static unsigned int table[256]; /* 保证多线程安全 */ if (!table_created) { /* 利用lock-free的思想创建一模一样的表 */ UTF8CreateLengthTable(table); /* 标记,之后的线程将不会重复创建该表 */ table_created = 1; } /* 这里使用查表法是因为考虑到这个表会被放入CPU的Cache */ for (i = 0; i < length;) { k = table[s[i]]; if (k == 0) break; for (j = 1; j < k; j++) if (table[s[i + j]]) return false; i += j; } return i == length; } bool ValidateGB2312Stream(const void* stream, unsigned length) { /* * 根据http://zh.wikipedia.org/zh-cn/Gb2312: * 01-09区为特殊符号。 * 16-55区为一级汉字,按拼音排序。 * 56-87区为二级汉字,按部首/笔画排序。 * 10-15区及88-94区则未有编码。 * * 每个汉字及符号以两个字节来表示。第一个字节称为“高位字节”,第二个字节称为“低位字节”。 * “高位字节”使用了0xA1-0xF7(把01-87区的区号加上0xA0),“低位字节”使用了0xA1-0xFE(把01-94加上 0xA0)。 * 由于一级汉字从16区起始,汉字区的“高位字节”的范围是0xB0-0xF7,“低位字节”的范围是0xA1-0xFE, * 占用的码位是 72*94=6768。其中有5个空位是D7FA-D7FE。 */ unsigned char* s = (unsigned char*)stream; unsigned char* e = s + length; for (; s < e; s++) { if (*s < 0x80) continue; if (*s < 0xA1 || 0xE7 < *s) break; if (++s == e) return false; if (*s < 0xA1 || 0xFE < *s) break; } return s == e; } bool ValidateGBKStream(const void* stream, unsigned length) { unsigned char* s = (unsigned char*)stream; unsigned char* e = s + length; for (; s < e; s++) { if (*s < 0x80) continue; if (*s < 0x81 || 0xFE < *s) break; if (++s == e) return false; if (*s < 0x40 || 0xFE < *s) break; } return s == e; } bool ValidateBIG5Stream(const void* stream, unsigned length) { unsigned char* s = (unsigned char*)stream; unsigned char* e = s + length; for (; s < e; s++) { if (*s < 0x80) continue; if (*s < 0xA1 || 0xF9 < *s) break; if (++s == e) return false; if (*s < 0x40 || 0xFE < *s || (0x7E < *s && *s < 0xA1)) break; } return s == e; } std::string StringTrimLeft(const char *input) { std::string output = input; StringTrimLeft(output); return output; } std::string StringTrimRight(const char *input) { std::string output = input; StringTrimRight(output); return output; } std::string StringTrim(const char *input) /* both left and right */ { std::string output = input; StringTrim(output); return output; } std::string& StringTrimLeft(std::string &input) { StringTrimLeftT(input); return input; } std::string& StringTrimRight(std::string &input) { StringTrimRightT(input); return input; } std::string& StringTrim(std::string &input) /* both left and right */ { StringTrimT(input); return input; } std::wstring StringTrimLeft(const wchar_t *input) { std::wstring output = input; StringTrimLeft(output); return output; } std::wstring StringTrimRight(const wchar_t *input) { std::wstring output = input; StringTrimRight(output); return output; } std::wstring StringTrim(const wchar_t *input) /* both left and right */ { std::wstring output = input; StringTrim(output); return output; } std::wstring& StringTrimLeft(std::wstring &input) { StringTrimLeftT(input); return input; } std::wstring& StringTrimRight(std::wstring &input) { StringTrimRightT(input); return input; } std::wstring& StringTrim(std::wstring &input) /* both left and right */ { StringTrimT(input); return input; } } // namespace nbase