诸暨麻将添加redis
25개 이상의 토픽을 선택하실 수 없습니다. Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

2402 lines
85 KiB

  1. // Protocol Buffers - Google's data interchange format
  2. // Copyright 2008 Google Inc. All rights reserved.
  3. // https://developers.google.com/protocol-buffers/
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // from google3/strings/strutil.cc
  31. #include <google/protobuf/stubs/strutil.h>
  32. #include <errno.h>
  33. #include <float.h> // FLT_DIG and DBL_DIG
  34. #include <limits.h>
  35. #include <stdio.h>
  36. #include <cmath>
  37. #include <iterator>
  38. #include <limits>
  39. #include <google/protobuf/stubs/logging.h>
  40. #include <google/protobuf/stubs/stl_util.h>
  41. #include <google/protobuf/io/strtod.h>
  42. #ifdef _WIN32
  43. // MSVC has only _snprintf, not snprintf.
  44. //
  45. // MinGW has both snprintf and _snprintf, but they appear to be different
  46. // functions. The former is buggy. When invoked like so:
  47. // char buffer[32];
  48. // snprintf(buffer, 32, "%.*g\n", FLT_DIG, 1.23e10f);
  49. // it prints "1.23000e+10". This is plainly wrong: %g should never print
  50. // trailing zeros after the decimal point. For some reason this bug only
  51. // occurs with some input values, not all. In any case, _snprintf does the
  52. // right thing, so we use it.
  53. #define snprintf _snprintf
  54. #endif
  55. namespace google {
  56. namespace protobuf {
  57. // These are defined as macros on some platforms. #undef them so that we can
  58. // redefine them.
  59. #undef isxdigit
  60. #undef isprint
  61. // The definitions of these in ctype.h change based on locale. Since our
  62. // string manipulation is all in relation to the protocol buffer and C++
  63. // languages, we always want to use the C locale. So, we re-define these
  64. // exactly as we want them.
  65. inline bool isxdigit(char c) {
  66. return ('0' <= c && c <= '9') ||
  67. ('a' <= c && c <= 'f') ||
  68. ('A' <= c && c <= 'F');
  69. }
  70. inline bool isprint(char c) {
  71. return c >= 0x20 && c <= 0x7E;
  72. }
  73. // ----------------------------------------------------------------------
  74. // ReplaceCharacters
  75. // Replaces any occurrence of the character 'remove' (or the characters
  76. // in 'remove') with the character 'replacewith'.
  77. // ----------------------------------------------------------------------
  78. void ReplaceCharacters(string *s, const char *remove, char replacewith) {
  79. const char *str_start = s->c_str();
  80. const char *str = str_start;
  81. for (str = strpbrk(str, remove);
  82. str != nullptr;
  83. str = strpbrk(str + 1, remove)) {
  84. (*s)[str - str_start] = replacewith;
  85. }
  86. }
  87. void StripWhitespace(string* str) {
  88. int str_length = str->length();
  89. // Strip off leading whitespace.
  90. int first = 0;
  91. while (first < str_length && ascii_isspace(str->at(first))) {
  92. ++first;
  93. }
  94. // If entire string is white space.
  95. if (first == str_length) {
  96. str->clear();
  97. return;
  98. }
  99. if (first > 0) {
  100. str->erase(0, first);
  101. str_length -= first;
  102. }
  103. // Strip off trailing whitespace.
  104. int last = str_length - 1;
  105. while (last >= 0 && ascii_isspace(str->at(last))) {
  106. --last;
  107. }
  108. if (last != (str_length - 1) && last >= 0) {
  109. str->erase(last + 1, string::npos);
  110. }
  111. }
  112. // ----------------------------------------------------------------------
  113. // StringReplace()
  114. // Replace the "old" pattern with the "new" pattern in a string,
  115. // and append the result to "res". If replace_all is false,
  116. // it only replaces the first instance of "old."
  117. // ----------------------------------------------------------------------
  118. void StringReplace(const string& s, const string& oldsub,
  119. const string& newsub, bool replace_all,
  120. string* res) {
  121. if (oldsub.empty()) {
  122. res->append(s); // if empty, append the given string.
  123. return;
  124. }
  125. string::size_type start_pos = 0;
  126. string::size_type pos;
  127. do {
  128. pos = s.find(oldsub, start_pos);
  129. if (pos == string::npos) {
  130. break;
  131. }
  132. res->append(s, start_pos, pos - start_pos);
  133. res->append(newsub);
  134. start_pos = pos + oldsub.size(); // start searching again after the "old"
  135. } while (replace_all);
  136. res->append(s, start_pos, s.length() - start_pos);
  137. }
  138. // ----------------------------------------------------------------------
  139. // StringReplace()
  140. // Give me a string and two patterns "old" and "new", and I replace
  141. // the first instance of "old" in the string with "new", if it
  142. // exists. If "global" is true; call this repeatedly until it
  143. // fails. RETURN a new string, regardless of whether the replacement
  144. // happened or not.
  145. // ----------------------------------------------------------------------
  146. string StringReplace(const string& s, const string& oldsub,
  147. const string& newsub, bool replace_all) {
  148. string ret;
  149. StringReplace(s, oldsub, newsub, replace_all, &ret);
  150. return ret;
  151. }
  152. // ----------------------------------------------------------------------
  153. // SplitStringUsing()
  154. // Split a string using a character delimiter. Append the components
  155. // to 'result'.
  156. //
  157. // Note: For multi-character delimiters, this routine will split on *ANY* of
  158. // the characters in the string, not the entire string as a single delimiter.
  159. // ----------------------------------------------------------------------
  160. template <typename ITR>
  161. static inline
  162. void SplitStringToIteratorUsing(const string& full,
  163. const char* delim,
  164. ITR& result) {
  165. // Optimize the common case where delim is a single character.
  166. if (delim[0] != '\0' && delim[1] == '\0') {
  167. char c = delim[0];
  168. const char* p = full.data();
  169. const char* end = p + full.size();
  170. while (p != end) {
  171. if (*p == c) {
  172. ++p;
  173. } else {
  174. const char* start = p;
  175. while (++p != end && *p != c);
  176. *result++ = string(start, p - start);
  177. }
  178. }
  179. return;
  180. }
  181. string::size_type begin_index, end_index;
  182. begin_index = full.find_first_not_of(delim);
  183. while (begin_index != string::npos) {
  184. end_index = full.find_first_of(delim, begin_index);
  185. if (end_index == string::npos) {
  186. *result++ = full.substr(begin_index);
  187. return;
  188. }
  189. *result++ = full.substr(begin_index, (end_index - begin_index));
  190. begin_index = full.find_first_not_of(delim, end_index);
  191. }
  192. }
  193. void SplitStringUsing(const string& full,
  194. const char* delim,
  195. std::vector<string>* result) {
  196. std::back_insert_iterator< std::vector<string> > it(*result);
  197. SplitStringToIteratorUsing(full, delim, it);
  198. }
  199. // Split a string using a character delimiter. Append the components
  200. // to 'result'. If there are consecutive delimiters, this function
  201. // will return corresponding empty strings. The string is split into
  202. // at most the specified number of pieces greedily. This means that the
  203. // last piece may possibly be split further. To split into as many pieces
  204. // as possible, specify 0 as the number of pieces.
  205. //
  206. // If "full" is the empty string, yields an empty string as the only value.
  207. //
  208. // If "pieces" is negative for some reason, it returns the whole string
  209. // ----------------------------------------------------------------------
  210. template <typename StringType, typename ITR>
  211. static inline
  212. void SplitStringToIteratorAllowEmpty(const StringType& full,
  213. const char* delim,
  214. int pieces,
  215. ITR& result) {
  216. string::size_type begin_index, end_index;
  217. begin_index = 0;
  218. for (int i = 0; (i < pieces-1) || (pieces == 0); i++) {
  219. end_index = full.find_first_of(delim, begin_index);
  220. if (end_index == string::npos) {
  221. *result++ = full.substr(begin_index);
  222. return;
  223. }
  224. *result++ = full.substr(begin_index, (end_index - begin_index));
  225. begin_index = end_index + 1;
  226. }
  227. *result++ = full.substr(begin_index);
  228. }
  229. void SplitStringAllowEmpty(const string& full, const char* delim,
  230. std::vector<string>* result) {
  231. std::back_insert_iterator<std::vector<string> > it(*result);
  232. SplitStringToIteratorAllowEmpty(full, delim, 0, it);
  233. }
  234. // ----------------------------------------------------------------------
  235. // JoinStrings()
  236. // This merges a vector of string components with delim inserted
  237. // as separaters between components.
  238. //
  239. // ----------------------------------------------------------------------
  240. template <class ITERATOR>
  241. static void JoinStringsIterator(const ITERATOR& start,
  242. const ITERATOR& end,
  243. const char* delim,
  244. string* result) {
  245. GOOGLE_CHECK(result != nullptr);
  246. result->clear();
  247. int delim_length = strlen(delim);
  248. // Precompute resulting length so we can reserve() memory in one shot.
  249. int length = 0;
  250. for (ITERATOR iter = start; iter != end; ++iter) {
  251. if (iter != start) {
  252. length += delim_length;
  253. }
  254. length += iter->size();
  255. }
  256. result->reserve(length);
  257. // Now combine everything.
  258. for (ITERATOR iter = start; iter != end; ++iter) {
  259. if (iter != start) {
  260. result->append(delim, delim_length);
  261. }
  262. result->append(iter->data(), iter->size());
  263. }
  264. }
  265. void JoinStrings(const std::vector<string>& components,
  266. const char* delim,
  267. string * result) {
  268. JoinStringsIterator(components.begin(), components.end(), delim, result);
  269. }
  270. // ----------------------------------------------------------------------
  271. // UnescapeCEscapeSequences()
  272. // This does all the unescaping that C does: \ooo, \r, \n, etc
  273. // Returns length of resulting string.
  274. // The implementation of \x parses any positive number of hex digits,
  275. // but it is an error if the value requires more than 8 bits, and the
  276. // result is truncated to 8 bits.
  277. //
  278. // The second call stores its errors in a supplied string vector.
  279. // If the string vector pointer is nullptr, it reports the errors with LOG().
  280. // ----------------------------------------------------------------------
  281. #define IS_OCTAL_DIGIT(c) (((c) >= '0') && ((c) <= '7'))
  282. // Protocol buffers doesn't ever care about errors, but I don't want to remove
  283. // the code.
  284. #define LOG_STRING(LEVEL, VECTOR) GOOGLE_LOG_IF(LEVEL, false)
  285. int UnescapeCEscapeSequences(const char* source, char* dest) {
  286. return UnescapeCEscapeSequences(source, dest, nullptr);
  287. }
  288. int UnescapeCEscapeSequences(const char* source, char* dest,
  289. std::vector<string> *errors) {
  290. GOOGLE_DCHECK(errors == nullptr) << "Error reporting not implemented.";
  291. char* d = dest;
  292. const char* p = source;
  293. // Small optimization for case where source = dest and there's no escaping
  294. while ( p == d && *p != '\0' && *p != '\\' )
  295. p++, d++;
  296. while (*p != '\0') {
  297. if (*p != '\\') {
  298. *d++ = *p++;
  299. } else {
  300. switch ( *++p ) { // skip past the '\\'
  301. case '\0':
  302. LOG_STRING(ERROR, errors) << "String cannot end with \\";
  303. *d = '\0';
  304. return d - dest; // we're done with p
  305. case 'a': *d++ = '\a'; break;
  306. case 'b': *d++ = '\b'; break;
  307. case 'f': *d++ = '\f'; break;
  308. case 'n': *d++ = '\n'; break;
  309. case 'r': *d++ = '\r'; break;
  310. case 't': *d++ = '\t'; break;
  311. case 'v': *d++ = '\v'; break;
  312. case '\\': *d++ = '\\'; break;
  313. case '?': *d++ = '\?'; break; // \? Who knew?
  314. case '\'': *d++ = '\''; break;
  315. case '"': *d++ = '\"'; break;
  316. case '0': case '1': case '2': case '3': // octal digit: 1 to 3 digits
  317. case '4': case '5': case '6': case '7': {
  318. char ch = *p - '0';
  319. if ( IS_OCTAL_DIGIT(p[1]) )
  320. ch = ch * 8 + *++p - '0';
  321. if ( IS_OCTAL_DIGIT(p[1]) ) // safe (and easy) to do this twice
  322. ch = ch * 8 + *++p - '0'; // now points at last digit
  323. *d++ = ch;
  324. break;
  325. }
  326. case 'x': case 'X': {
  327. if (!isxdigit(p[1])) {
  328. if (p[1] == '\0') {
  329. LOG_STRING(ERROR, errors) << "String cannot end with \\x";
  330. } else {
  331. LOG_STRING(ERROR, errors) <<
  332. "\\x cannot be followed by non-hex digit: \\" << *p << p[1];
  333. }
  334. break;
  335. }
  336. unsigned int ch = 0;
  337. const char *hex_start = p;
  338. while (isxdigit(p[1])) // arbitrarily many hex digits
  339. ch = (ch << 4) + hex_digit_to_int(*++p);
  340. if (ch > 0xFF)
  341. LOG_STRING(ERROR, errors) << "Value of " <<
  342. "\\" << string(hex_start, p+1-hex_start) << " exceeds 8 bits";
  343. *d++ = ch;
  344. break;
  345. }
  346. #if 0 // TODO(kenton): Support \u and \U? Requires runetochar().
  347. case 'u': {
  348. // \uhhhh => convert 4 hex digits to UTF-8
  349. char32 rune = 0;
  350. const char *hex_start = p;
  351. for (int i = 0; i < 4; ++i) {
  352. if (isxdigit(p[1])) { // Look one char ahead.
  353. rune = (rune << 4) + hex_digit_to_int(*++p); // Advance p.
  354. } else {
  355. LOG_STRING(ERROR, errors)
  356. << "\\u must be followed by 4 hex digits: \\"
  357. << string(hex_start, p+1-hex_start);
  358. break;
  359. }
  360. }
  361. d += runetochar(d, &rune);
  362. break;
  363. }
  364. case 'U': {
  365. // \Uhhhhhhhh => convert 8 hex digits to UTF-8
  366. char32 rune = 0;
  367. const char *hex_start = p;
  368. for (int i = 0; i < 8; ++i) {
  369. if (isxdigit(p[1])) { // Look one char ahead.
  370. // Don't change rune until we're sure this
  371. // is within the Unicode limit, but do advance p.
  372. char32 newrune = (rune << 4) + hex_digit_to_int(*++p);
  373. if (newrune > 0x10FFFF) {
  374. LOG_STRING(ERROR, errors)
  375. << "Value of \\"
  376. << string(hex_start, p + 1 - hex_start)
  377. << " exceeds Unicode limit (0x10FFFF)";
  378. break;
  379. } else {
  380. rune = newrune;
  381. }
  382. } else {
  383. LOG_STRING(ERROR, errors)
  384. << "\\U must be followed by 8 hex digits: \\"
  385. << string(hex_start, p+1-hex_start);
  386. break;
  387. }
  388. }
  389. d += runetochar(d, &rune);
  390. break;
  391. }
  392. #endif
  393. default:
  394. LOG_STRING(ERROR, errors) << "Unknown escape sequence: \\" << *p;
  395. }
  396. p++; // read past letter we escaped
  397. }
  398. }
  399. *d = '\0';
  400. return d - dest;
  401. }
  402. // ----------------------------------------------------------------------
  403. // UnescapeCEscapeString()
  404. // This does the same thing as UnescapeCEscapeSequences, but creates
  405. // a new string. The caller does not need to worry about allocating
  406. // a dest buffer. This should be used for non performance critical
  407. // tasks such as printing debug messages. It is safe for src and dest
  408. // to be the same.
  409. //
  410. // The second call stores its errors in a supplied string vector.
  411. // If the string vector pointer is nullptr, it reports the errors with LOG().
  412. //
  413. // In the first and second calls, the length of dest is returned. In the
  414. // the third call, the new string is returned.
  415. // ----------------------------------------------------------------------
  416. int UnescapeCEscapeString(const string& src, string* dest) {
  417. return UnescapeCEscapeString(src, dest, nullptr);
  418. }
  419. int UnescapeCEscapeString(const string& src, string* dest,
  420. std::vector<string> *errors) {
  421. std::unique_ptr<char[]> unescaped(new char[src.size() + 1]);
  422. int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), errors);
  423. GOOGLE_CHECK(dest);
  424. dest->assign(unescaped.get(), len);
  425. return len;
  426. }
  427. string UnescapeCEscapeString(const string& src) {
  428. std::unique_ptr<char[]> unescaped(new char[src.size() + 1]);
  429. int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), nullptr);
  430. return string(unescaped.get(), len);
  431. }
  432. // ----------------------------------------------------------------------
  433. // CEscapeString()
  434. // CHexEscapeString()
  435. // Copies 'src' to 'dest', escaping dangerous characters using
  436. // C-style escape sequences. This is very useful for preparing query
  437. // flags. 'src' and 'dest' should not overlap. The 'Hex' version uses
  438. // hexadecimal rather than octal sequences.
  439. // Returns the number of bytes written to 'dest' (not including the \0)
  440. // or -1 if there was insufficient space.
  441. //
  442. // Currently only \n, \r, \t, ", ', \ and !isprint() chars are escaped.
  443. // ----------------------------------------------------------------------
  444. int CEscapeInternal(const char* src, int src_len, char* dest,
  445. int dest_len, bool use_hex, bool utf8_safe) {
  446. const char* src_end = src + src_len;
  447. int used = 0;
  448. bool last_hex_escape = false; // true if last output char was \xNN
  449. for (; src < src_end; src++) {
  450. if (dest_len - used < 2) // Need space for two letter escape
  451. return -1;
  452. bool is_hex_escape = false;
  453. switch (*src) {
  454. case '\n': dest[used++] = '\\'; dest[used++] = 'n'; break;
  455. case '\r': dest[used++] = '\\'; dest[used++] = 'r'; break;
  456. case '\t': dest[used++] = '\\'; dest[used++] = 't'; break;
  457. case '\"': dest[used++] = '\\'; dest[used++] = '\"'; break;
  458. case '\'': dest[used++] = '\\'; dest[used++] = '\''; break;
  459. case '\\': dest[used++] = '\\'; dest[used++] = '\\'; break;
  460. default:
  461. // Note that if we emit \xNN and the src character after that is a hex
  462. // digit then that digit must be escaped too to prevent it being
  463. // interpreted as part of the character code by C.
  464. if ((!utf8_safe || static_cast<uint8>(*src) < 0x80) &&
  465. (!isprint(*src) ||
  466. (last_hex_escape && isxdigit(*src)))) {
  467. if (dest_len - used < 4) // need space for 4 letter escape
  468. return -1;
  469. sprintf(dest + used, (use_hex ? "\\x%02x" : "\\%03o"),
  470. static_cast<uint8>(*src));
  471. is_hex_escape = use_hex;
  472. used += 4;
  473. } else {
  474. dest[used++] = *src; break;
  475. }
  476. }
  477. last_hex_escape = is_hex_escape;
  478. }
  479. if (dest_len - used < 1) // make sure that there is room for \0
  480. return -1;
  481. dest[used] = '\0'; // doesn't count towards return value though
  482. return used;
  483. }
  484. // Calculates the length of the C-style escaped version of 'src'.
  485. // Assumes that non-printable characters are escaped using octal sequences, and
  486. // that UTF-8 bytes are not handled specially.
  487. static inline size_t CEscapedLength(StringPiece src) {
  488. static char c_escaped_len[256] = {
  489. 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 4, 4, 2, 4, 4, // \t, \n, \r
  490. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  491. 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // ", '
  492. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // '0'..'9'
  493. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'A'..'O'
  494. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, // 'P'..'Z', '\'
  495. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'a'..'o'
  496. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, // 'p'..'z', DEL
  497. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  498. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  499. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  500. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  501. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  502. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  503. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  504. 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
  505. };
  506. size_t escaped_len = 0;
  507. for (int i = 0; i < src.size(); ++i) {
  508. unsigned char c = static_cast<unsigned char>(src[i]);
  509. escaped_len += c_escaped_len[c];
  510. }
  511. return escaped_len;
  512. }
  513. // ----------------------------------------------------------------------
  514. // Escapes 'src' using C-style escape sequences, and appends the escaped string
  515. // to 'dest'. This version is faster than calling CEscapeInternal as it computes
  516. // the required space using a lookup table, and also does not do any special
  517. // handling for Hex or UTF-8 characters.
  518. // ----------------------------------------------------------------------
  519. void CEscapeAndAppend(StringPiece src, string* dest) {
  520. size_t escaped_len = CEscapedLength(src);
  521. if (escaped_len == src.size()) {
  522. dest->append(src.data(), src.size());
  523. return;
  524. }
  525. size_t cur_dest_len = dest->size();
  526. dest->resize(cur_dest_len + escaped_len);
  527. char* append_ptr = &(*dest)[cur_dest_len];
  528. for (int i = 0; i < src.size(); ++i) {
  529. unsigned char c = static_cast<unsigned char>(src[i]);
  530. switch (c) {
  531. case '\n': *append_ptr++ = '\\'; *append_ptr++ = 'n'; break;
  532. case '\r': *append_ptr++ = '\\'; *append_ptr++ = 'r'; break;
  533. case '\t': *append_ptr++ = '\\'; *append_ptr++ = 't'; break;
  534. case '\"': *append_ptr++ = '\\'; *append_ptr++ = '\"'; break;
  535. case '\'': *append_ptr++ = '\\'; *append_ptr++ = '\''; break;
  536. case '\\': *append_ptr++ = '\\'; *append_ptr++ = '\\'; break;
  537. default:
  538. if (!isprint(c)) {
  539. *append_ptr++ = '\\';
  540. *append_ptr++ = '0' + c / 64;
  541. *append_ptr++ = '0' + (c % 64) / 8;
  542. *append_ptr++ = '0' + c % 8;
  543. } else {
  544. *append_ptr++ = c;
  545. }
  546. break;
  547. }
  548. }
  549. }
  550. string CEscape(const string& src) {
  551. string dest;
  552. CEscapeAndAppend(src, &dest);
  553. return dest;
  554. }
  555. namespace strings {
  556. string Utf8SafeCEscape(const string& src) {
  557. const int dest_length = src.size() * 4 + 1; // Maximum possible expansion
  558. std::unique_ptr<char[]> dest(new char[dest_length]);
  559. const int len = CEscapeInternal(src.data(), src.size(),
  560. dest.get(), dest_length, false, true);
  561. GOOGLE_DCHECK_GE(len, 0);
  562. return string(dest.get(), len);
  563. }
  564. string CHexEscape(const string& src) {
  565. const int dest_length = src.size() * 4 + 1; // Maximum possible expansion
  566. std::unique_ptr<char[]> dest(new char[dest_length]);
  567. const int len = CEscapeInternal(src.data(), src.size(),
  568. dest.get(), dest_length, true, false);
  569. GOOGLE_DCHECK_GE(len, 0);
  570. return string(dest.get(), len);
  571. }
  572. } // namespace strings
  573. // ----------------------------------------------------------------------
  574. // strto32_adaptor()
  575. // strtou32_adaptor()
  576. // Implementation of strto[u]l replacements that have identical
  577. // overflow and underflow characteristics for both ILP-32 and LP-64
  578. // platforms, including errno preservation in error-free calls.
  579. // ----------------------------------------------------------------------
  580. int32 strto32_adaptor(const char *nptr, char **endptr, int base) {
  581. const int saved_errno = errno;
  582. errno = 0;
  583. const long result = strtol(nptr, endptr, base);
  584. if (errno == ERANGE && result == LONG_MIN) {
  585. return kint32min;
  586. } else if (errno == ERANGE && result == LONG_MAX) {
  587. return kint32max;
  588. } else if (errno == 0 && result < kint32min) {
  589. errno = ERANGE;
  590. return kint32min;
  591. } else if (errno == 0 && result > kint32max) {
  592. errno = ERANGE;
  593. return kint32max;
  594. }
  595. if (errno == 0)
  596. errno = saved_errno;
  597. return static_cast<int32>(result);
  598. }
  599. uint32 strtou32_adaptor(const char *nptr, char **endptr, int base) {
  600. const int saved_errno = errno;
  601. errno = 0;
  602. const unsigned long result = strtoul(nptr, endptr, base);
  603. if (errno == ERANGE && result == ULONG_MAX) {
  604. return kuint32max;
  605. } else if (errno == 0 && result > kuint32max) {
  606. errno = ERANGE;
  607. return kuint32max;
  608. }
  609. if (errno == 0)
  610. errno = saved_errno;
  611. return static_cast<uint32>(result);
  612. }
  613. inline bool safe_parse_sign(string* text /*inout*/,
  614. bool* negative_ptr /*output*/) {
  615. const char* start = text->data();
  616. const char* end = start + text->size();
  617. // Consume whitespace.
  618. while (start < end && (start[0] == ' ')) {
  619. ++start;
  620. }
  621. while (start < end && (end[-1] == ' ')) {
  622. --end;
  623. }
  624. if (start >= end) {
  625. return false;
  626. }
  627. // Consume sign.
  628. *negative_ptr = (start[0] == '-');
  629. if (*negative_ptr || start[0] == '+') {
  630. ++start;
  631. if (start >= end) {
  632. return false;
  633. }
  634. }
  635. *text = text->substr(start - text->data(), end - start);
  636. return true;
  637. }
  638. template<typename IntType>
  639. bool safe_parse_positive_int(
  640. string text, IntType* value_p) {
  641. int base = 10;
  642. IntType value = 0;
  643. const IntType vmax = std::numeric_limits<IntType>::max();
  644. assert(vmax > 0);
  645. assert(vmax >= base);
  646. const IntType vmax_over_base = vmax / base;
  647. const char* start = text.data();
  648. const char* end = start + text.size();
  649. // loop over digits
  650. for (; start < end; ++start) {
  651. unsigned char c = static_cast<unsigned char>(start[0]);
  652. int digit = c - '0';
  653. if (digit >= base || digit < 0) {
  654. *value_p = value;
  655. return false;
  656. }
  657. if (value > vmax_over_base) {
  658. *value_p = vmax;
  659. return false;
  660. }
  661. value *= base;
  662. if (value > vmax - digit) {
  663. *value_p = vmax;
  664. return false;
  665. }
  666. value += digit;
  667. }
  668. *value_p = value;
  669. return true;
  670. }
  671. template<typename IntType>
  672. bool safe_parse_negative_int(
  673. const string& text, IntType* value_p) {
  674. int base = 10;
  675. IntType value = 0;
  676. const IntType vmin = std::numeric_limits<IntType>::min();
  677. assert(vmin < 0);
  678. assert(vmin <= 0 - base);
  679. IntType vmin_over_base = vmin / base;
  680. // 2003 c++ standard [expr.mul]
  681. // "... the sign of the remainder is implementation-defined."
  682. // Although (vmin/base)*base + vmin%base is always vmin.
  683. // 2011 c++ standard tightens the spec but we cannot rely on it.
  684. if (vmin % base > 0) {
  685. vmin_over_base += 1;
  686. }
  687. const char* start = text.data();
  688. const char* end = start + text.size();
  689. // loop over digits
  690. for (; start < end; ++start) {
  691. unsigned char c = static_cast<unsigned char>(start[0]);
  692. int digit = c - '0';
  693. if (digit >= base || digit < 0) {
  694. *value_p = value;
  695. return false;
  696. }
  697. if (value < vmin_over_base) {
  698. *value_p = vmin;
  699. return false;
  700. }
  701. value *= base;
  702. if (value < vmin + digit) {
  703. *value_p = vmin;
  704. return false;
  705. }
  706. value -= digit;
  707. }
  708. *value_p = value;
  709. return true;
  710. }
  711. template<typename IntType>
  712. bool safe_int_internal(string text, IntType* value_p) {
  713. *value_p = 0;
  714. bool negative;
  715. if (!safe_parse_sign(&text, &negative)) {
  716. return false;
  717. }
  718. if (!negative) {
  719. return safe_parse_positive_int(text, value_p);
  720. } else {
  721. return safe_parse_negative_int(text, value_p);
  722. }
  723. }
  724. template<typename IntType>
  725. bool safe_uint_internal(string text, IntType* value_p) {
  726. *value_p = 0;
  727. bool negative;
  728. if (!safe_parse_sign(&text, &negative) || negative) {
  729. return false;
  730. }
  731. return safe_parse_positive_int(text, value_p);
  732. }
  733. // ----------------------------------------------------------------------
  734. // FastIntToBuffer()
  735. // FastInt64ToBuffer()
  736. // FastHexToBuffer()
  737. // FastHex64ToBuffer()
  738. // FastHex32ToBuffer()
  739. // ----------------------------------------------------------------------
  740. // Offset into buffer where FastInt64ToBuffer places the end of string
  741. // null character. Also used by FastInt64ToBufferLeft.
  742. static const int kFastInt64ToBufferOffset = 21;
  743. char *FastInt64ToBuffer(int64 i, char* buffer) {
  744. // We could collapse the positive and negative sections, but that
  745. // would be slightly slower for positive numbers...
  746. // 22 bytes is enough to store -2**64, -18446744073709551616.
  747. char* p = buffer + kFastInt64ToBufferOffset;
  748. *p-- = '\0';
  749. if (i >= 0) {
  750. do {
  751. *p-- = '0' + i % 10;
  752. i /= 10;
  753. } while (i > 0);
  754. return p + 1;
  755. } else {
  756. // On different platforms, % and / have different behaviors for
  757. // negative numbers, so we need to jump through hoops to make sure
  758. // we don't divide negative numbers.
  759. if (i > -10) {
  760. i = -i;
  761. *p-- = '0' + i;
  762. *p = '-';
  763. return p;
  764. } else {
  765. // Make sure we aren't at MIN_INT, in which case we can't say i = -i
  766. i = i + 10;
  767. i = -i;
  768. *p-- = '0' + i % 10;
  769. // Undo what we did a moment ago
  770. i = i / 10 + 1;
  771. do {
  772. *p-- = '0' + i % 10;
  773. i /= 10;
  774. } while (i > 0);
  775. *p = '-';
  776. return p;
  777. }
  778. }
  779. }
  780. // Offset into buffer where FastInt32ToBuffer places the end of string
  781. // null character. Also used by FastInt32ToBufferLeft
  782. static const int kFastInt32ToBufferOffset = 11;
  783. // Yes, this is a duplicate of FastInt64ToBuffer. But, we need this for the
  784. // compiler to generate 32 bit arithmetic instructions. It's much faster, at
  785. // least with 32 bit binaries.
  786. char *FastInt32ToBuffer(int32 i, char* buffer) {
  787. // We could collapse the positive and negative sections, but that
  788. // would be slightly slower for positive numbers...
  789. // 12 bytes is enough to store -2**32, -4294967296.
  790. char* p = buffer + kFastInt32ToBufferOffset;
  791. *p-- = '\0';
  792. if (i >= 0) {
  793. do {
  794. *p-- = '0' + i % 10;
  795. i /= 10;
  796. } while (i > 0);
  797. return p + 1;
  798. } else {
  799. // On different platforms, % and / have different behaviors for
  800. // negative numbers, so we need to jump through hoops to make sure
  801. // we don't divide negative numbers.
  802. if (i > -10) {
  803. i = -i;
  804. *p-- = '0' + i;
  805. *p = '-';
  806. return p;
  807. } else {
  808. // Make sure we aren't at MIN_INT, in which case we can't say i = -i
  809. i = i + 10;
  810. i = -i;
  811. *p-- = '0' + i % 10;
  812. // Undo what we did a moment ago
  813. i = i / 10 + 1;
  814. do {
  815. *p-- = '0' + i % 10;
  816. i /= 10;
  817. } while (i > 0);
  818. *p = '-';
  819. return p;
  820. }
  821. }
  822. }
  823. char *FastHexToBuffer(int i, char* buffer) {
  824. GOOGLE_CHECK(i >= 0) << "FastHexToBuffer() wants non-negative integers, not " << i;
  825. static const char *hexdigits = "0123456789abcdef";
  826. char *p = buffer + 21;
  827. *p-- = '\0';
  828. do {
  829. *p-- = hexdigits[i & 15]; // mod by 16
  830. i >>= 4; // divide by 16
  831. } while (i > 0);
  832. return p + 1;
  833. }
  834. char *InternalFastHexToBuffer(uint64 value, char* buffer, int num_byte) {
  835. static const char *hexdigits = "0123456789abcdef";
  836. buffer[num_byte] = '\0';
  837. for (int i = num_byte - 1; i >= 0; i--) {
  838. #ifdef _M_X64
  839. // MSVC x64 platform has a bug optimizing the uint32(value) in the #else
  840. // block. Given that the uint32 cast was to improve performance on 32-bit
  841. // platforms, we use 64-bit '&' directly.
  842. buffer[i] = hexdigits[value & 0xf];
  843. #else
  844. buffer[i] = hexdigits[uint32(value) & 0xf];
  845. #endif
  846. value >>= 4;
  847. }
  848. return buffer;
  849. }
  850. char *FastHex64ToBuffer(uint64 value, char* buffer) {
  851. return InternalFastHexToBuffer(value, buffer, 16);
  852. }
  853. char *FastHex32ToBuffer(uint32 value, char* buffer) {
  854. return InternalFastHexToBuffer(value, buffer, 8);
  855. }
  856. // ----------------------------------------------------------------------
  857. // FastInt32ToBufferLeft()
  858. // FastUInt32ToBufferLeft()
  859. // FastInt64ToBufferLeft()
  860. // FastUInt64ToBufferLeft()
  861. //
  862. // Like the Fast*ToBuffer() functions above, these are intended for speed.
  863. // Unlike the Fast*ToBuffer() functions, however, these functions write
  864. // their output to the beginning of the buffer (hence the name, as the
  865. // output is left-aligned). The caller is responsible for ensuring that
  866. // the buffer has enough space to hold the output.
  867. //
  868. // Returns a pointer to the end of the string (i.e. the null character
  869. // terminating the string).
  870. // ----------------------------------------------------------------------
  871. static const char two_ASCII_digits[100][2] = {
  872. {'0','0'}, {'0','1'}, {'0','2'}, {'0','3'}, {'0','4'},
  873. {'0','5'}, {'0','6'}, {'0','7'}, {'0','8'}, {'0','9'},
  874. {'1','0'}, {'1','1'}, {'1','2'}, {'1','3'}, {'1','4'},
  875. {'1','5'}, {'1','6'}, {'1','7'}, {'1','8'}, {'1','9'},
  876. {'2','0'}, {'2','1'}, {'2','2'}, {'2','3'}, {'2','4'},
  877. {'2','5'}, {'2','6'}, {'2','7'}, {'2','8'}, {'2','9'},
  878. {'3','0'}, {'3','1'}, {'3','2'}, {'3','3'}, {'3','4'},
  879. {'3','5'}, {'3','6'}, {'3','7'}, {'3','8'}, {'3','9'},
  880. {'4','0'}, {'4','1'}, {'4','2'}, {'4','3'}, {'4','4'},
  881. {'4','5'}, {'4','6'}, {'4','7'}, {'4','8'}, {'4','9'},
  882. {'5','0'}, {'5','1'}, {'5','2'}, {'5','3'}, {'5','4'},
  883. {'5','5'}, {'5','6'}, {'5','7'}, {'5','8'}, {'5','9'},
  884. {'6','0'}, {'6','1'}, {'6','2'}, {'6','3'}, {'6','4'},
  885. {'6','5'}, {'6','6'}, {'6','7'}, {'6','8'}, {'6','9'},
  886. {'7','0'}, {'7','1'}, {'7','2'}, {'7','3'}, {'7','4'},
  887. {'7','5'}, {'7','6'}, {'7','7'}, {'7','8'}, {'7','9'},
  888. {'8','0'}, {'8','1'}, {'8','2'}, {'8','3'}, {'8','4'},
  889. {'8','5'}, {'8','6'}, {'8','7'}, {'8','8'}, {'8','9'},
  890. {'9','0'}, {'9','1'}, {'9','2'}, {'9','3'}, {'9','4'},
  891. {'9','5'}, {'9','6'}, {'9','7'}, {'9','8'}, {'9','9'}
  892. };
  893. char* FastUInt32ToBufferLeft(uint32 u, char* buffer) {
  894. uint32 digits;
  895. const char *ASCII_digits = nullptr;
  896. // The idea of this implementation is to trim the number of divides to as few
  897. // as possible by using multiplication and subtraction rather than mod (%),
  898. // and by outputting two digits at a time rather than one.
  899. // The huge-number case is first, in the hopes that the compiler will output
  900. // that case in one branch-free block of code, and only output conditional
  901. // branches into it from below.
  902. if (u >= 1000000000) { // >= 1,000,000,000
  903. digits = u / 100000000; // 100,000,000
  904. ASCII_digits = two_ASCII_digits[digits];
  905. buffer[0] = ASCII_digits[0];
  906. buffer[1] = ASCII_digits[1];
  907. buffer += 2;
  908. sublt100_000_000:
  909. u -= digits * 100000000; // 100,000,000
  910. lt100_000_000:
  911. digits = u / 1000000; // 1,000,000
  912. ASCII_digits = two_ASCII_digits[digits];
  913. buffer[0] = ASCII_digits[0];
  914. buffer[1] = ASCII_digits[1];
  915. buffer += 2;
  916. sublt1_000_000:
  917. u -= digits * 1000000; // 1,000,000
  918. lt1_000_000:
  919. digits = u / 10000; // 10,000
  920. ASCII_digits = two_ASCII_digits[digits];
  921. buffer[0] = ASCII_digits[0];
  922. buffer[1] = ASCII_digits[1];
  923. buffer += 2;
  924. sublt10_000:
  925. u -= digits * 10000; // 10,000
  926. lt10_000:
  927. digits = u / 100;
  928. ASCII_digits = two_ASCII_digits[digits];
  929. buffer[0] = ASCII_digits[0];
  930. buffer[1] = ASCII_digits[1];
  931. buffer += 2;
  932. sublt100:
  933. u -= digits * 100;
  934. lt100:
  935. digits = u;
  936. ASCII_digits = two_ASCII_digits[digits];
  937. buffer[0] = ASCII_digits[0];
  938. buffer[1] = ASCII_digits[1];
  939. buffer += 2;
  940. done:
  941. *buffer = 0;
  942. return buffer;
  943. }
  944. if (u < 100) {
  945. digits = u;
  946. if (u >= 10) goto lt100;
  947. *buffer++ = '0' + digits;
  948. goto done;
  949. }
  950. if (u < 10000) { // 10,000
  951. if (u >= 1000) goto lt10_000;
  952. digits = u / 100;
  953. *buffer++ = '0' + digits;
  954. goto sublt100;
  955. }
  956. if (u < 1000000) { // 1,000,000
  957. if (u >= 100000) goto lt1_000_000;
  958. digits = u / 10000; // 10,000
  959. *buffer++ = '0' + digits;
  960. goto sublt10_000;
  961. }
  962. if (u < 100000000) { // 100,000,000
  963. if (u >= 10000000) goto lt100_000_000;
  964. digits = u / 1000000; // 1,000,000
  965. *buffer++ = '0' + digits;
  966. goto sublt1_000_000;
  967. }
  968. // we already know that u < 1,000,000,000
  969. digits = u / 100000000; // 100,000,000
  970. *buffer++ = '0' + digits;
  971. goto sublt100_000_000;
  972. }
  973. char* FastInt32ToBufferLeft(int32 i, char* buffer) {
  974. uint32 u = 0;
  975. if (i < 0) {
  976. *buffer++ = '-';
  977. u -= i;
  978. } else {
  979. u = i;
  980. }
  981. return FastUInt32ToBufferLeft(u, buffer);
  982. }
  983. char* FastUInt64ToBufferLeft(uint64 u64, char* buffer) {
  984. int digits;
  985. const char *ASCII_digits = nullptr;
  986. uint32 u = static_cast<uint32>(u64);
  987. if (u == u64) return FastUInt32ToBufferLeft(u, buffer);
  988. uint64 top_11_digits = u64 / 1000000000;
  989. buffer = FastUInt64ToBufferLeft(top_11_digits, buffer);
  990. u = u64 - (top_11_digits * 1000000000);
  991. digits = u / 10000000; // 10,000,000
  992. GOOGLE_DCHECK_LT(digits, 100);
  993. ASCII_digits = two_ASCII_digits[digits];
  994. buffer[0] = ASCII_digits[0];
  995. buffer[1] = ASCII_digits[1];
  996. buffer += 2;
  997. u -= digits * 10000000; // 10,000,000
  998. digits = u / 100000; // 100,000
  999. ASCII_digits = two_ASCII_digits[digits];
  1000. buffer[0] = ASCII_digits[0];
  1001. buffer[1] = ASCII_digits[1];
  1002. buffer += 2;
  1003. u -= digits * 100000; // 100,000
  1004. digits = u / 1000; // 1,000
  1005. ASCII_digits = two_ASCII_digits[digits];
  1006. buffer[0] = ASCII_digits[0];
  1007. buffer[1] = ASCII_digits[1];
  1008. buffer += 2;
  1009. u -= digits * 1000; // 1,000
  1010. digits = u / 10;
  1011. ASCII_digits = two_ASCII_digits[digits];
  1012. buffer[0] = ASCII_digits[0];
  1013. buffer[1] = ASCII_digits[1];
  1014. buffer += 2;
  1015. u -= digits * 10;
  1016. digits = u;
  1017. *buffer++ = '0' + digits;
  1018. *buffer = 0;
  1019. return buffer;
  1020. }
  1021. char* FastInt64ToBufferLeft(int64 i, char* buffer) {
  1022. uint64 u = 0;
  1023. if (i < 0) {
  1024. *buffer++ = '-';
  1025. u -= i;
  1026. } else {
  1027. u = i;
  1028. }
  1029. return FastUInt64ToBufferLeft(u, buffer);
  1030. }
  1031. // ----------------------------------------------------------------------
  1032. // SimpleItoa()
  1033. // Description: converts an integer to a string.
  1034. //
  1035. // Return value: string
  1036. // ----------------------------------------------------------------------
  1037. string SimpleItoa(int i) {
  1038. char buffer[kFastToBufferSize];
  1039. return (sizeof(i) == 4) ?
  1040. FastInt32ToBuffer(i, buffer) :
  1041. FastInt64ToBuffer(i, buffer);
  1042. }
  1043. string SimpleItoa(unsigned int i) {
  1044. char buffer[kFastToBufferSize];
  1045. return string(buffer, (sizeof(i) == 4) ?
  1046. FastUInt32ToBufferLeft(i, buffer) :
  1047. FastUInt64ToBufferLeft(i, buffer));
  1048. }
  1049. string SimpleItoa(long i) {
  1050. char buffer[kFastToBufferSize];
  1051. return (sizeof(i) == 4) ?
  1052. FastInt32ToBuffer(i, buffer) :
  1053. FastInt64ToBuffer(i, buffer);
  1054. }
  1055. string SimpleItoa(unsigned long i) {
  1056. char buffer[kFastToBufferSize];
  1057. return string(buffer, (sizeof(i) == 4) ?
  1058. FastUInt32ToBufferLeft(i, buffer) :
  1059. FastUInt64ToBufferLeft(i, buffer));
  1060. }
  1061. string SimpleItoa(long long i) {
  1062. char buffer[kFastToBufferSize];
  1063. return (sizeof(i) == 4) ?
  1064. FastInt32ToBuffer(i, buffer) :
  1065. FastInt64ToBuffer(i, buffer);
  1066. }
  1067. string SimpleItoa(unsigned long long i) {
  1068. char buffer[kFastToBufferSize];
  1069. return string(buffer, (sizeof(i) == 4) ?
  1070. FastUInt32ToBufferLeft(i, buffer) :
  1071. FastUInt64ToBufferLeft(i, buffer));
  1072. }
  1073. // ----------------------------------------------------------------------
  1074. // SimpleDtoa()
  1075. // SimpleFtoa()
  1076. // DoubleToBuffer()
  1077. // FloatToBuffer()
  1078. // We want to print the value without losing precision, but we also do
  1079. // not want to print more digits than necessary. This turns out to be
  1080. // trickier than it sounds. Numbers like 0.2 cannot be represented
  1081. // exactly in binary. If we print 0.2 with a very large precision,
  1082. // e.g. "%.50g", we get "0.2000000000000000111022302462515654042363167".
  1083. // On the other hand, if we set the precision too low, we lose
  1084. // significant digits when printing numbers that actually need them.
  1085. // It turns out there is no precision value that does the right thing
  1086. // for all numbers.
  1087. //
  1088. // Our strategy is to first try printing with a precision that is never
  1089. // over-precise, then parse the result with strtod() to see if it
  1090. // matches. If not, we print again with a precision that will always
  1091. // give a precise result, but may use more digits than necessary.
  1092. //
  1093. // An arguably better strategy would be to use the algorithm described
  1094. // in "How to Print Floating-Point Numbers Accurately" by Steele &
  1095. // White, e.g. as implemented by David M. Gay's dtoa(). It turns out,
  1096. // however, that the following implementation is about as fast as
  1097. // DMG's code. Furthermore, DMG's code locks mutexes, which means it
  1098. // will not scale well on multi-core machines. DMG's code is slightly
  1099. // more accurate (in that it will never use more digits than
  1100. // necessary), but this is probably irrelevant for most users.
  1101. //
  1102. // Rob Pike and Ken Thompson also have an implementation of dtoa() in
  1103. // third_party/fmt/fltfmt.cc. Their implementation is similar to this
  1104. // one in that it makes guesses and then uses strtod() to check them.
  1105. // Their implementation is faster because they use their own code to
  1106. // generate the digits in the first place rather than use snprintf(),
  1107. // thus avoiding format string parsing overhead. However, this makes
  1108. // it considerably more complicated than the following implementation,
  1109. // and it is embedded in a larger library. If speed turns out to be
  1110. // an issue, we could re-implement this in terms of their
  1111. // implementation.
  1112. // ----------------------------------------------------------------------
  1113. string SimpleDtoa(double value) {
  1114. char buffer[kDoubleToBufferSize];
  1115. return DoubleToBuffer(value, buffer);
  1116. }
  1117. string SimpleFtoa(float value) {
  1118. char buffer[kFloatToBufferSize];
  1119. return FloatToBuffer(value, buffer);
  1120. }
  1121. static inline bool IsValidFloatChar(char c) {
  1122. return ('0' <= c && c <= '9') ||
  1123. c == 'e' || c == 'E' ||
  1124. c == '+' || c == '-';
  1125. }
  1126. void DelocalizeRadix(char* buffer) {
  1127. // Fast check: if the buffer has a normal decimal point, assume no
  1128. // translation is needed.
  1129. if (strchr(buffer, '.') != nullptr) return;
  1130. // Find the first unknown character.
  1131. while (IsValidFloatChar(*buffer)) ++buffer;
  1132. if (*buffer == '\0') {
  1133. // No radix character found.
  1134. return;
  1135. }
  1136. // We are now pointing at the locale-specific radix character. Replace it
  1137. // with '.'.
  1138. *buffer = '.';
  1139. ++buffer;
  1140. if (!IsValidFloatChar(*buffer) && *buffer != '\0') {
  1141. // It appears the radix was a multi-byte character. We need to remove the
  1142. // extra bytes.
  1143. char* target = buffer;
  1144. do { ++buffer; } while (!IsValidFloatChar(*buffer) && *buffer != '\0');
  1145. memmove(target, buffer, strlen(buffer) + 1);
  1146. }
  1147. }
  1148. char* DoubleToBuffer(double value, char* buffer) {
  1149. // DBL_DIG is 15 for IEEE-754 doubles, which are used on almost all
  1150. // platforms these days. Just in case some system exists where DBL_DIG
  1151. // is significantly larger -- and risks overflowing our buffer -- we have
  1152. // this assert.
  1153. GOOGLE_COMPILE_ASSERT(DBL_DIG < 20, DBL_DIG_is_too_big);
  1154. if (value == std::numeric_limits<double>::infinity()) {
  1155. strcpy(buffer, "inf");
  1156. return buffer;
  1157. } else if (value == -std::numeric_limits<double>::infinity()) {
  1158. strcpy(buffer, "-inf");
  1159. return buffer;
  1160. } else if (std::isnan(value)) {
  1161. strcpy(buffer, "nan");
  1162. return buffer;
  1163. }
  1164. int snprintf_result =
  1165. snprintf(buffer, kDoubleToBufferSize, "%.*g", DBL_DIG, value);
  1166. // The snprintf should never overflow because the buffer is significantly
  1167. // larger than the precision we asked for.
  1168. GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kDoubleToBufferSize);
  1169. // We need to make parsed_value volatile in order to force the compiler to
  1170. // write it out to the stack. Otherwise, it may keep the value in a
  1171. // register, and if it does that, it may keep it as a long double instead
  1172. // of a double. This long double may have extra bits that make it compare
  1173. // unequal to "value" even though it would be exactly equal if it were
  1174. // truncated to a double.
  1175. volatile double parsed_value = io::NoLocaleStrtod(buffer, nullptr);
  1176. if (parsed_value != value) {
  1177. int snprintf_result =
  1178. snprintf(buffer, kDoubleToBufferSize, "%.*g", DBL_DIG+2, value);
  1179. // Should never overflow; see above.
  1180. GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kDoubleToBufferSize);
  1181. }
  1182. DelocalizeRadix(buffer);
  1183. return buffer;
  1184. }
  1185. static int memcasecmp(const char *s1, const char *s2, size_t len) {
  1186. const unsigned char *us1 = reinterpret_cast<const unsigned char *>(s1);
  1187. const unsigned char *us2 = reinterpret_cast<const unsigned char *>(s2);
  1188. for ( int i = 0; i < len; i++ ) {
  1189. const int diff =
  1190. static_cast<int>(static_cast<unsigned char>(ascii_tolower(us1[i]))) -
  1191. static_cast<int>(static_cast<unsigned char>(ascii_tolower(us2[i])));
  1192. if (diff != 0) return diff;
  1193. }
  1194. return 0;
  1195. }
  1196. inline bool CaseEqual(StringPiece s1, StringPiece s2) {
  1197. if (s1.size() != s2.size()) return false;
  1198. return memcasecmp(s1.data(), s2.data(), s1.size()) == 0;
  1199. }
  1200. bool safe_strtob(StringPiece str, bool* value) {
  1201. GOOGLE_CHECK(value != nullptr) << "nullptr output boolean given.";
  1202. if (CaseEqual(str, "true") || CaseEqual(str, "t") ||
  1203. CaseEqual(str, "yes") || CaseEqual(str, "y") ||
  1204. CaseEqual(str, "1")) {
  1205. *value = true;
  1206. return true;
  1207. }
  1208. if (CaseEqual(str, "false") || CaseEqual(str, "f") ||
  1209. CaseEqual(str, "no") || CaseEqual(str, "n") ||
  1210. CaseEqual(str, "0")) {
  1211. *value = false;
  1212. return true;
  1213. }
  1214. return false;
  1215. }
  1216. bool safe_strtof(const char* str, float* value) {
  1217. char* endptr;
  1218. errno = 0; // errno only gets set on errors
  1219. #if defined(_WIN32) || defined (__hpux) // has no strtof()
  1220. *value = io::NoLocaleStrtod(str, &endptr);
  1221. #else
  1222. *value = strtof(str, &endptr);
  1223. #endif
  1224. return *str != 0 && *endptr == 0 && errno == 0;
  1225. }
  1226. bool safe_strtod(const char* str, double* value) {
  1227. char* endptr;
  1228. *value = io::NoLocaleStrtod(str, &endptr);
  1229. if (endptr != str) {
  1230. while (ascii_isspace(*endptr)) ++endptr;
  1231. }
  1232. // Ignore range errors from strtod. The values it
  1233. // returns on underflow and overflow are the right
  1234. // fallback in a robust setting.
  1235. return *str != '\0' && *endptr == '\0';
  1236. }
  1237. bool safe_strto32(const string& str, int32* value) {
  1238. return safe_int_internal(str, value);
  1239. }
  1240. bool safe_strtou32(const string& str, uint32* value) {
  1241. return safe_uint_internal(str, value);
  1242. }
  1243. bool safe_strto64(const string& str, int64* value) {
  1244. return safe_int_internal(str, value);
  1245. }
  1246. bool safe_strtou64(const string& str, uint64* value) {
  1247. return safe_uint_internal(str, value);
  1248. }
  1249. char* FloatToBuffer(float value, char* buffer) {
  1250. // FLT_DIG is 6 for IEEE-754 floats, which are used on almost all
  1251. // platforms these days. Just in case some system exists where FLT_DIG
  1252. // is significantly larger -- and risks overflowing our buffer -- we have
  1253. // this assert.
  1254. GOOGLE_COMPILE_ASSERT(FLT_DIG < 10, FLT_DIG_is_too_big);
  1255. if (value == std::numeric_limits<double>::infinity()) {
  1256. strcpy(buffer, "inf");
  1257. return buffer;
  1258. } else if (value == -std::numeric_limits<double>::infinity()) {
  1259. strcpy(buffer, "-inf");
  1260. return buffer;
  1261. } else if (std::isnan(value)) {
  1262. strcpy(buffer, "nan");
  1263. return buffer;
  1264. }
  1265. int snprintf_result =
  1266. snprintf(buffer, kFloatToBufferSize, "%.*g", FLT_DIG, value);
  1267. // The snprintf should never overflow because the buffer is significantly
  1268. // larger than the precision we asked for.
  1269. GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kFloatToBufferSize);
  1270. float parsed_value;
  1271. if (!safe_strtof(buffer, &parsed_value) || parsed_value != value) {
  1272. int snprintf_result =
  1273. snprintf(buffer, kFloatToBufferSize, "%.*g", FLT_DIG+3, value);
  1274. // Should never overflow; see above.
  1275. GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kFloatToBufferSize);
  1276. }
  1277. DelocalizeRadix(buffer);
  1278. return buffer;
  1279. }
  1280. namespace strings {
  1281. AlphaNum::AlphaNum(strings::Hex hex) {
  1282. char *const end = &digits[kFastToBufferSize];
  1283. char *writer = end;
  1284. uint64 value = hex.value;
  1285. uint64 width = hex.spec;
  1286. // We accomplish minimum width by OR'ing in 0x10000 to the user's value,
  1287. // where 0x10000 is the smallest hex number that is as wide as the user
  1288. // asked for.
  1289. uint64 mask = ((static_cast<uint64>(1) << (width - 1) * 4)) | value;
  1290. static const char hexdigits[] = "0123456789abcdef";
  1291. do {
  1292. *--writer = hexdigits[value & 0xF];
  1293. value >>= 4;
  1294. mask >>= 4;
  1295. } while (mask != 0);
  1296. piece_data_ = writer;
  1297. piece_size_ = end - writer;
  1298. }
  1299. } // namespace strings
  1300. // ----------------------------------------------------------------------
  1301. // StrCat()
  1302. // This merges the given strings or integers, with no delimiter. This
  1303. // is designed to be the fastest possible way to construct a string out
  1304. // of a mix of raw C strings, C++ strings, and integer values.
  1305. // ----------------------------------------------------------------------
  1306. // Append is merely a version of memcpy that returns the address of the byte
  1307. // after the area just overwritten. It comes in multiple flavors to minimize
  1308. // call overhead.
  1309. static char *Append1(char *out, const AlphaNum &x) {
  1310. memcpy(out, x.data(), x.size());
  1311. return out + x.size();
  1312. }
  1313. static char *Append2(char *out, const AlphaNum &x1, const AlphaNum &x2) {
  1314. memcpy(out, x1.data(), x1.size());
  1315. out += x1.size();
  1316. memcpy(out, x2.data(), x2.size());
  1317. return out + x2.size();
  1318. }
  1319. static char *Append4(char *out,
  1320. const AlphaNum &x1, const AlphaNum &x2,
  1321. const AlphaNum &x3, const AlphaNum &x4) {
  1322. memcpy(out, x1.data(), x1.size());
  1323. out += x1.size();
  1324. memcpy(out, x2.data(), x2.size());
  1325. out += x2.size();
  1326. memcpy(out, x3.data(), x3.size());
  1327. out += x3.size();
  1328. memcpy(out, x4.data(), x4.size());
  1329. return out + x4.size();
  1330. }
  1331. string StrCat(const AlphaNum &a, const AlphaNum &b) {
  1332. string result;
  1333. result.resize(a.size() + b.size());
  1334. char *const begin = &*result.begin();
  1335. char *out = Append2(begin, a, b);
  1336. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1337. return result;
  1338. }
  1339. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c) {
  1340. string result;
  1341. result.resize(a.size() + b.size() + c.size());
  1342. char *const begin = &*result.begin();
  1343. char *out = Append2(begin, a, b);
  1344. out = Append1(out, c);
  1345. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1346. return result;
  1347. }
  1348. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1349. const AlphaNum &d) {
  1350. string result;
  1351. result.resize(a.size() + b.size() + c.size() + d.size());
  1352. char *const begin = &*result.begin();
  1353. char *out = Append4(begin, a, b, c, d);
  1354. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1355. return result;
  1356. }
  1357. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1358. const AlphaNum &d, const AlphaNum &e) {
  1359. string result;
  1360. result.resize(a.size() + b.size() + c.size() + d.size() + e.size());
  1361. char *const begin = &*result.begin();
  1362. char *out = Append4(begin, a, b, c, d);
  1363. out = Append1(out, e);
  1364. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1365. return result;
  1366. }
  1367. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1368. const AlphaNum &d, const AlphaNum &e, const AlphaNum &f) {
  1369. string result;
  1370. result.resize(a.size() + b.size() + c.size() + d.size() + e.size() +
  1371. f.size());
  1372. char *const begin = &*result.begin();
  1373. char *out = Append4(begin, a, b, c, d);
  1374. out = Append2(out, e, f);
  1375. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1376. return result;
  1377. }
  1378. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1379. const AlphaNum &d, const AlphaNum &e, const AlphaNum &f,
  1380. const AlphaNum &g) {
  1381. string result;
  1382. result.resize(a.size() + b.size() + c.size() + d.size() + e.size() +
  1383. f.size() + g.size());
  1384. char *const begin = &*result.begin();
  1385. char *out = Append4(begin, a, b, c, d);
  1386. out = Append2(out, e, f);
  1387. out = Append1(out, g);
  1388. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1389. return result;
  1390. }
  1391. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1392. const AlphaNum &d, const AlphaNum &e, const AlphaNum &f,
  1393. const AlphaNum &g, const AlphaNum &h) {
  1394. string result;
  1395. result.resize(a.size() + b.size() + c.size() + d.size() + e.size() +
  1396. f.size() + g.size() + h.size());
  1397. char *const begin = &*result.begin();
  1398. char *out = Append4(begin, a, b, c, d);
  1399. out = Append4(out, e, f, g, h);
  1400. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1401. return result;
  1402. }
  1403. string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
  1404. const AlphaNum &d, const AlphaNum &e, const AlphaNum &f,
  1405. const AlphaNum &g, const AlphaNum &h, const AlphaNum &i) {
  1406. string result;
  1407. result.resize(a.size() + b.size() + c.size() + d.size() + e.size() +
  1408. f.size() + g.size() + h.size() + i.size());
  1409. char *const begin = &*result.begin();
  1410. char *out = Append4(begin, a, b, c, d);
  1411. out = Append4(out, e, f, g, h);
  1412. out = Append1(out, i);
  1413. GOOGLE_DCHECK_EQ(out, begin + result.size());
  1414. return result;
  1415. }
  1416. // It's possible to call StrAppend with a char * pointer that is partway into
  1417. // the string we're appending to. However the results of this are random.
  1418. // Therefore, check for this in debug mode. Use unsigned math so we only have
  1419. // to do one comparison.
  1420. #define GOOGLE_DCHECK_NO_OVERLAP(dest, src) \
  1421. GOOGLE_DCHECK_GT(uintptr_t((src).data() - (dest).data()), \
  1422. uintptr_t((dest).size()))
  1423. void StrAppend(string *result, const AlphaNum &a) {
  1424. GOOGLE_DCHECK_NO_OVERLAP(*result, a);
  1425. result->append(a.data(), a.size());
  1426. }
  1427. void StrAppend(string *result, const AlphaNum &a, const AlphaNum &b) {
  1428. GOOGLE_DCHECK_NO_OVERLAP(*result, a);
  1429. GOOGLE_DCHECK_NO_OVERLAP(*result, b);
  1430. string::size_type old_size = result->size();
  1431. result->resize(old_size + a.size() + b.size());
  1432. char *const begin = &*result->begin();
  1433. char *out = Append2(begin + old_size, a, b);
  1434. GOOGLE_DCHECK_EQ(out, begin + result->size());
  1435. }
  1436. void StrAppend(string *result,
  1437. const AlphaNum &a, const AlphaNum &b, const AlphaNum &c) {
  1438. GOOGLE_DCHECK_NO_OVERLAP(*result, a);
  1439. GOOGLE_DCHECK_NO_OVERLAP(*result, b);
  1440. GOOGLE_DCHECK_NO_OVERLAP(*result, c);
  1441. string::size_type old_size = result->size();
  1442. result->resize(old_size + a.size() + b.size() + c.size());
  1443. char *const begin = &*result->begin();
  1444. char *out = Append2(begin + old_size, a, b);
  1445. out = Append1(out, c);
  1446. GOOGLE_DCHECK_EQ(out, begin + result->size());
  1447. }
  1448. void StrAppend(string *result,
  1449. const AlphaNum &a, const AlphaNum &b,
  1450. const AlphaNum &c, const AlphaNum &d) {
  1451. GOOGLE_DCHECK_NO_OVERLAP(*result, a);
  1452. GOOGLE_DCHECK_NO_OVERLAP(*result, b);
  1453. GOOGLE_DCHECK_NO_OVERLAP(*result, c);
  1454. GOOGLE_DCHECK_NO_OVERLAP(*result, d);
  1455. string::size_type old_size = result->size();
  1456. result->resize(old_size + a.size() + b.size() + c.size() + d.size());
  1457. char *const begin = &*result->begin();
  1458. char *out = Append4(begin + old_size, a, b, c, d);
  1459. GOOGLE_DCHECK_EQ(out, begin + result->size());
  1460. }
  1461. int GlobalReplaceSubstring(const string& substring,
  1462. const string& replacement,
  1463. string* s) {
  1464. GOOGLE_CHECK(s != nullptr);
  1465. if (s->empty() || substring.empty())
  1466. return 0;
  1467. string tmp;
  1468. int num_replacements = 0;
  1469. int pos = 0;
  1470. for (int match_pos = s->find(substring.data(), pos, substring.length());
  1471. match_pos != string::npos;
  1472. pos = match_pos + substring.length(),
  1473. match_pos = s->find(substring.data(), pos, substring.length())) {
  1474. ++num_replacements;
  1475. // Append the original content before the match.
  1476. tmp.append(*s, pos, match_pos - pos);
  1477. // Append the replacement for the match.
  1478. tmp.append(replacement.begin(), replacement.end());
  1479. }
  1480. // Append the content after the last match. If no replacements were made, the
  1481. // original string is left untouched.
  1482. if (num_replacements > 0) {
  1483. tmp.append(*s, pos, s->length() - pos);
  1484. s->swap(tmp);
  1485. }
  1486. return num_replacements;
  1487. }
  1488. int CalculateBase64EscapedLen(int input_len, bool do_padding) {
  1489. // Base64 encodes three bytes of input at a time. If the input is not
  1490. // divisible by three, we pad as appropriate.
  1491. //
  1492. // (from http://tools.ietf.org/html/rfc3548)
  1493. // Special processing is performed if fewer than 24 bits are available
  1494. // at the end of the data being encoded. A full encoding quantum is
  1495. // always completed at the end of a quantity. When fewer than 24 input
  1496. // bits are available in an input group, zero bits are added (on the
  1497. // right) to form an integral number of 6-bit groups. Padding at the
  1498. // end of the data is performed using the '=' character. Since all base
  1499. // 64 input is an integral number of octets, only the following cases
  1500. // can arise:
  1501. // Base64 encodes each three bytes of input into four bytes of output.
  1502. int len = (input_len / 3) * 4;
  1503. if (input_len % 3 == 0) {
  1504. // (from http://tools.ietf.org/html/rfc3548)
  1505. // (1) the final quantum of encoding input is an integral multiple of 24
  1506. // bits; here, the final unit of encoded output will be an integral
  1507. // multiple of 4 characters with no "=" padding,
  1508. } else if (input_len % 3 == 1) {
  1509. // (from http://tools.ietf.org/html/rfc3548)
  1510. // (2) the final quantum of encoding input is exactly 8 bits; here, the
  1511. // final unit of encoded output will be two characters followed by two
  1512. // "=" padding characters, or
  1513. len += 2;
  1514. if (do_padding) {
  1515. len += 2;
  1516. }
  1517. } else { // (input_len % 3 == 2)
  1518. // (from http://tools.ietf.org/html/rfc3548)
  1519. // (3) the final quantum of encoding input is exactly 16 bits; here, the
  1520. // final unit of encoded output will be three characters followed by one
  1521. // "=" padding character.
  1522. len += 3;
  1523. if (do_padding) {
  1524. len += 1;
  1525. }
  1526. }
  1527. assert(len >= input_len); // make sure we didn't overflow
  1528. return len;
  1529. }
  1530. // Base64Escape does padding, so this calculation includes padding.
  1531. int CalculateBase64EscapedLen(int input_len) {
  1532. return CalculateBase64EscapedLen(input_len, true);
  1533. }
  1534. // ----------------------------------------------------------------------
  1535. // int Base64Unescape() - base64 decoder
  1536. // int Base64Escape() - base64 encoder
  1537. // int WebSafeBase64Unescape() - Google's variation of base64 decoder
  1538. // int WebSafeBase64Escape() - Google's variation of base64 encoder
  1539. //
  1540. // Check out
  1541. // http://tools.ietf.org/html/rfc2045 for formal description, but what we
  1542. // care about is that...
  1543. // Take the encoded stuff in groups of 4 characters and turn each
  1544. // character into a code 0 to 63 thus:
  1545. // A-Z map to 0 to 25
  1546. // a-z map to 26 to 51
  1547. // 0-9 map to 52 to 61
  1548. // +(- for WebSafe) maps to 62
  1549. // /(_ for WebSafe) maps to 63
  1550. // There will be four numbers, all less than 64 which can be represented
  1551. // by a 6 digit binary number (aaaaaa, bbbbbb, cccccc, dddddd respectively).
  1552. // Arrange the 6 digit binary numbers into three bytes as such:
  1553. // aaaaaabb bbbbcccc ccdddddd
  1554. // Equals signs (one or two) are used at the end of the encoded block to
  1555. // indicate that the text was not an integer multiple of three bytes long.
  1556. // ----------------------------------------------------------------------
  1557. int Base64UnescapeInternal(const char *src_param, int szsrc,
  1558. char *dest, int szdest,
  1559. const signed char* unbase64) {
  1560. static const char kPad64Equals = '=';
  1561. static const char kPad64Dot = '.';
  1562. int decode = 0;
  1563. int destidx = 0;
  1564. int state = 0;
  1565. unsigned int ch = 0;
  1566. unsigned int temp = 0;
  1567. // If "char" is signed by default, using *src as an array index results in
  1568. // accessing negative array elements. Treat the input as a pointer to
  1569. // unsigned char to avoid this.
  1570. const unsigned char *src = reinterpret_cast<const unsigned char*>(src_param);
  1571. // The GET_INPUT macro gets the next input character, skipping
  1572. // over any whitespace, and stopping when we reach the end of the
  1573. // string or when we read any non-data character. The arguments are
  1574. // an arbitrary identifier (used as a label for goto) and the number
  1575. // of data bytes that must remain in the input to avoid aborting the
  1576. // loop.
  1577. #define GET_INPUT(label, remain) \
  1578. label: \
  1579. --szsrc; \
  1580. ch = *src++; \
  1581. decode = unbase64[ch]; \
  1582. if (decode < 0) { \
  1583. if (ascii_isspace(ch) && szsrc >= remain) \
  1584. goto label; \
  1585. state = 4 - remain; \
  1586. break; \
  1587. }
  1588. // if dest is null, we're just checking to see if it's legal input
  1589. // rather than producing output. (I suspect this could just be done
  1590. // with a regexp...). We duplicate the loop so this test can be
  1591. // outside it instead of in every iteration.
  1592. if (dest) {
  1593. // This loop consumes 4 input bytes and produces 3 output bytes
  1594. // per iteration. We can't know at the start that there is enough
  1595. // data left in the string for a full iteration, so the loop may
  1596. // break out in the middle; if so 'state' will be set to the
  1597. // number of input bytes read.
  1598. while (szsrc >= 4) {
  1599. // We'll start by optimistically assuming that the next four
  1600. // bytes of the string (src[0..3]) are four good data bytes
  1601. // (that is, no nulls, whitespace, padding chars, or illegal
  1602. // chars). We need to test src[0..2] for nulls individually
  1603. // before constructing temp to preserve the property that we
  1604. // never read past a null in the string (no matter how long
  1605. // szsrc claims the string is).
  1606. if (!src[0] || !src[1] || !src[2] ||
  1607. (temp = ((unsigned(unbase64[src[0]]) << 18) |
  1608. (unsigned(unbase64[src[1]]) << 12) |
  1609. (unsigned(unbase64[src[2]]) << 6) |
  1610. (unsigned(unbase64[src[3]])))) & 0x80000000) {
  1611. // Iff any of those four characters was bad (null, illegal,
  1612. // whitespace, padding), then temp's high bit will be set
  1613. // (because unbase64[] is -1 for all bad characters).
  1614. //
  1615. // We'll back up and resort to the slower decoder, which knows
  1616. // how to handle those cases.
  1617. GET_INPUT(first, 4);
  1618. temp = decode;
  1619. GET_INPUT(second, 3);
  1620. temp = (temp << 6) | decode;
  1621. GET_INPUT(third, 2);
  1622. temp = (temp << 6) | decode;
  1623. GET_INPUT(fourth, 1);
  1624. temp = (temp << 6) | decode;
  1625. } else {
  1626. // We really did have four good data bytes, so advance four
  1627. // characters in the string.
  1628. szsrc -= 4;
  1629. src += 4;
  1630. decode = -1;
  1631. ch = '\0';
  1632. }
  1633. // temp has 24 bits of input, so write that out as three bytes.
  1634. if (destidx+3 > szdest) return -1;
  1635. dest[destidx+2] = temp;
  1636. temp >>= 8;
  1637. dest[destidx+1] = temp;
  1638. temp >>= 8;
  1639. dest[destidx] = temp;
  1640. destidx += 3;
  1641. }
  1642. } else {
  1643. while (szsrc >= 4) {
  1644. if (!src[0] || !src[1] || !src[2] ||
  1645. (temp = ((unsigned(unbase64[src[0]]) << 18) |
  1646. (unsigned(unbase64[src[1]]) << 12) |
  1647. (unsigned(unbase64[src[2]]) << 6) |
  1648. (unsigned(unbase64[src[3]])))) & 0x80000000) {
  1649. GET_INPUT(first_no_dest, 4);
  1650. GET_INPUT(second_no_dest, 3);
  1651. GET_INPUT(third_no_dest, 2);
  1652. GET_INPUT(fourth_no_dest, 1);
  1653. } else {
  1654. szsrc -= 4;
  1655. src += 4;
  1656. decode = -1;
  1657. ch = '\0';
  1658. }
  1659. destidx += 3;
  1660. }
  1661. }
  1662. #undef GET_INPUT
  1663. // if the loop terminated because we read a bad character, return
  1664. // now.
  1665. if (decode < 0 && ch != '\0' &&
  1666. ch != kPad64Equals && ch != kPad64Dot && !ascii_isspace(ch))
  1667. return -1;
  1668. if (ch == kPad64Equals || ch == kPad64Dot) {
  1669. // if we stopped by hitting an '=' or '.', un-read that character -- we'll
  1670. // look at it again when we count to check for the proper number of
  1671. // equals signs at the end.
  1672. ++szsrc;
  1673. --src;
  1674. } else {
  1675. // This loop consumes 1 input byte per iteration. It's used to
  1676. // clean up the 0-3 input bytes remaining when the first, faster
  1677. // loop finishes. 'temp' contains the data from 'state' input
  1678. // characters read by the first loop.
  1679. while (szsrc > 0) {
  1680. --szsrc;
  1681. ch = *src++;
  1682. decode = unbase64[ch];
  1683. if (decode < 0) {
  1684. if (ascii_isspace(ch)) {
  1685. continue;
  1686. } else if (ch == '\0') {
  1687. break;
  1688. } else if (ch == kPad64Equals || ch == kPad64Dot) {
  1689. // back up one character; we'll read it again when we check
  1690. // for the correct number of pad characters at the end.
  1691. ++szsrc;
  1692. --src;
  1693. break;
  1694. } else {
  1695. return -1;
  1696. }
  1697. }
  1698. // Each input character gives us six bits of output.
  1699. temp = (temp << 6) | decode;
  1700. ++state;
  1701. if (state == 4) {
  1702. // If we've accumulated 24 bits of output, write that out as
  1703. // three bytes.
  1704. if (dest) {
  1705. if (destidx+3 > szdest) return -1;
  1706. dest[destidx+2] = temp;
  1707. temp >>= 8;
  1708. dest[destidx+1] = temp;
  1709. temp >>= 8;
  1710. dest[destidx] = temp;
  1711. }
  1712. destidx += 3;
  1713. state = 0;
  1714. temp = 0;
  1715. }
  1716. }
  1717. }
  1718. // Process the leftover data contained in 'temp' at the end of the input.
  1719. int expected_equals = 0;
  1720. switch (state) {
  1721. case 0:
  1722. // Nothing left over; output is a multiple of 3 bytes.
  1723. break;
  1724. case 1:
  1725. // Bad input; we have 6 bits left over.
  1726. return -1;
  1727. case 2:
  1728. // Produce one more output byte from the 12 input bits we have left.
  1729. if (dest) {
  1730. if (destidx+1 > szdest) return -1;
  1731. temp >>= 4;
  1732. dest[destidx] = temp;
  1733. }
  1734. ++destidx;
  1735. expected_equals = 2;
  1736. break;
  1737. case 3:
  1738. // Produce two more output bytes from the 18 input bits we have left.
  1739. if (dest) {
  1740. if (destidx+2 > szdest) return -1;
  1741. temp >>= 2;
  1742. dest[destidx+1] = temp;
  1743. temp >>= 8;
  1744. dest[destidx] = temp;
  1745. }
  1746. destidx += 2;
  1747. expected_equals = 1;
  1748. break;
  1749. default:
  1750. // state should have no other values at this point.
  1751. GOOGLE_LOG(FATAL) << "This can't happen; base64 decoder state = " << state;
  1752. }
  1753. // The remainder of the string should be all whitespace, mixed with
  1754. // exactly 0 equals signs, or exactly 'expected_equals' equals
  1755. // signs. (Always accepting 0 equals signs is a google extension
  1756. // not covered in the RFC, as is accepting dot as the pad character.)
  1757. int equals = 0;
  1758. while (szsrc > 0 && *src) {
  1759. if (*src == kPad64Equals || *src == kPad64Dot)
  1760. ++equals;
  1761. else if (!ascii_isspace(*src))
  1762. return -1;
  1763. --szsrc;
  1764. ++src;
  1765. }
  1766. return (equals == 0 || equals == expected_equals) ? destidx : -1;
  1767. }
  1768. // The arrays below were generated by the following code
  1769. // #include <sys/time.h>
  1770. // #include <stdlib.h>
  1771. // #include <string.h>
  1772. // #include <stdio.h>
  1773. // main()
  1774. // {
  1775. // static const char Base64[] =
  1776. // "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
  1777. // const char *pos;
  1778. // int idx, i, j;
  1779. // printf(" ");
  1780. // for (i = 0; i < 255; i += 8) {
  1781. // for (j = i; j < i + 8; j++) {
  1782. // pos = strchr(Base64, j);
  1783. // if ((pos == nullptr) || (j == 0))
  1784. // idx = -1;
  1785. // else
  1786. // idx = pos - Base64;
  1787. // if (idx == -1)
  1788. // printf(" %2d, ", idx);
  1789. // else
  1790. // printf(" %2d/""*%c*""/,", idx, j);
  1791. // }
  1792. // printf("\n ");
  1793. // }
  1794. // }
  1795. //
  1796. // where the value of "Base64[]" was replaced by one of the base-64 conversion
  1797. // tables from the functions below.
  1798. static const signed char kUnBase64[] = {
  1799. -1, -1, -1, -1, -1, -1, -1, -1,
  1800. -1, -1, -1, -1, -1, -1, -1, -1,
  1801. -1, -1, -1, -1, -1, -1, -1, -1,
  1802. -1, -1, -1, -1, -1, -1, -1, -1,
  1803. -1, -1, -1, -1, -1, -1, -1, -1,
  1804. -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */,
  1805. 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/,
  1806. 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1,
  1807. -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/,
  1808. 7/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/,
  1809. 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/,
  1810. 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1,
  1811. -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/,
  1812. 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/,
  1813. 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/,
  1814. 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1,
  1815. -1, -1, -1, -1, -1, -1, -1, -1,
  1816. -1, -1, -1, -1, -1, -1, -1, -1,
  1817. -1, -1, -1, -1, -1, -1, -1, -1,
  1818. -1, -1, -1, -1, -1, -1, -1, -1,
  1819. -1, -1, -1, -1, -1, -1, -1, -1,
  1820. -1, -1, -1, -1, -1, -1, -1, -1,
  1821. -1, -1, -1, -1, -1, -1, -1, -1,
  1822. -1, -1, -1, -1, -1, -1, -1, -1,
  1823. -1, -1, -1, -1, -1, -1, -1, -1,
  1824. -1, -1, -1, -1, -1, -1, -1, -1,
  1825. -1, -1, -1, -1, -1, -1, -1, -1,
  1826. -1, -1, -1, -1, -1, -1, -1, -1,
  1827. -1, -1, -1, -1, -1, -1, -1, -1,
  1828. -1, -1, -1, -1, -1, -1, -1, -1,
  1829. -1, -1, -1, -1, -1, -1, -1, -1,
  1830. -1, -1, -1, -1, -1, -1, -1, -1
  1831. };
  1832. static const signed char kUnWebSafeBase64[] = {
  1833. -1, -1, -1, -1, -1, -1, -1, -1,
  1834. -1, -1, -1, -1, -1, -1, -1, -1,
  1835. -1, -1, -1, -1, -1, -1, -1, -1,
  1836. -1, -1, -1, -1, -1, -1, -1, -1,
  1837. -1, -1, -1, -1, -1, -1, -1, -1,
  1838. -1, -1, -1, -1, -1, 62/*-*/, -1, -1,
  1839. 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/,
  1840. 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1,
  1841. -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/,
  1842. 7/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/,
  1843. 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/,
  1844. 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, 63/*_*/,
  1845. -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/,
  1846. 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/,
  1847. 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/,
  1848. 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1,
  1849. -1, -1, -1, -1, -1, -1, -1, -1,
  1850. -1, -1, -1, -1, -1, -1, -1, -1,
  1851. -1, -1, -1, -1, -1, -1, -1, -1,
  1852. -1, -1, -1, -1, -1, -1, -1, -1,
  1853. -1, -1, -1, -1, -1, -1, -1, -1,
  1854. -1, -1, -1, -1, -1, -1, -1, -1,
  1855. -1, -1, -1, -1, -1, -1, -1, -1,
  1856. -1, -1, -1, -1, -1, -1, -1, -1,
  1857. -1, -1, -1, -1, -1, -1, -1, -1,
  1858. -1, -1, -1, -1, -1, -1, -1, -1,
  1859. -1, -1, -1, -1, -1, -1, -1, -1,
  1860. -1, -1, -1, -1, -1, -1, -1, -1,
  1861. -1, -1, -1, -1, -1, -1, -1, -1,
  1862. -1, -1, -1, -1, -1, -1, -1, -1,
  1863. -1, -1, -1, -1, -1, -1, -1, -1,
  1864. -1, -1, -1, -1, -1, -1, -1, -1
  1865. };
  1866. int WebSafeBase64Unescape(const char *src, int szsrc, char *dest, int szdest) {
  1867. return Base64UnescapeInternal(src, szsrc, dest, szdest, kUnWebSafeBase64);
  1868. }
  1869. static bool Base64UnescapeInternal(const char* src, int slen, string* dest,
  1870. const signed char* unbase64) {
  1871. // Determine the size of the output string. Base64 encodes every 3 bytes into
  1872. // 4 characters. any leftover chars are added directly for good measure.
  1873. // This is documented in the base64 RFC: http://tools.ietf.org/html/rfc3548
  1874. const int dest_len = 3 * (slen / 4) + (slen % 4);
  1875. dest->resize(dest_len);
  1876. // We are getting the destination buffer by getting the beginning of the
  1877. // string and converting it into a char *.
  1878. const int len = Base64UnescapeInternal(src, slen, string_as_array(dest),
  1879. dest_len, unbase64);
  1880. if (len < 0) {
  1881. dest->clear();
  1882. return false;
  1883. }
  1884. // could be shorter if there was padding
  1885. GOOGLE_DCHECK_LE(len, dest_len);
  1886. dest->erase(len);
  1887. return true;
  1888. }
  1889. bool Base64Unescape(StringPiece src, string* dest) {
  1890. return Base64UnescapeInternal(src.data(), src.size(), dest, kUnBase64);
  1891. }
  1892. bool WebSafeBase64Unescape(StringPiece src, string* dest) {
  1893. return Base64UnescapeInternal(src.data(), src.size(), dest, kUnWebSafeBase64);
  1894. }
  1895. int Base64EscapeInternal(const unsigned char *src, int szsrc,
  1896. char *dest, int szdest, const char *base64,
  1897. bool do_padding) {
  1898. static const char kPad64 = '=';
  1899. if (szsrc <= 0) return 0;
  1900. if (szsrc * 4 > szdest * 3) return 0;
  1901. char *cur_dest = dest;
  1902. const unsigned char *cur_src = src;
  1903. char *limit_dest = dest + szdest;
  1904. const unsigned char *limit_src = src + szsrc;
  1905. // Three bytes of data encodes to four characters of cyphertext.
  1906. // So we can pump through three-byte chunks atomically.
  1907. while (cur_src < limit_src - 3) { // keep going as long as we have >= 32 bits
  1908. uint32 in = BigEndian::Load32(cur_src) >> 8;
  1909. cur_dest[0] = base64[in >> 18];
  1910. in &= 0x3FFFF;
  1911. cur_dest[1] = base64[in >> 12];
  1912. in &= 0xFFF;
  1913. cur_dest[2] = base64[in >> 6];
  1914. in &= 0x3F;
  1915. cur_dest[3] = base64[in];
  1916. cur_dest += 4;
  1917. cur_src += 3;
  1918. }
  1919. // To save time, we didn't update szdest or szsrc in the loop. So do it now.
  1920. szdest = limit_dest - cur_dest;
  1921. szsrc = limit_src - cur_src;
  1922. /* now deal with the tail (<=3 bytes) */
  1923. switch (szsrc) {
  1924. case 0:
  1925. // Nothing left; nothing more to do.
  1926. break;
  1927. case 1: {
  1928. // One byte left: this encodes to two characters, and (optionally)
  1929. // two pad characters to round out the four-character cypherblock.
  1930. if ((szdest -= 2) < 0) return 0;
  1931. uint32 in = cur_src[0];
  1932. cur_dest[0] = base64[in >> 2];
  1933. in &= 0x3;
  1934. cur_dest[1] = base64[in << 4];
  1935. cur_dest += 2;
  1936. if (do_padding) {
  1937. if ((szdest -= 2) < 0) return 0;
  1938. cur_dest[0] = kPad64;
  1939. cur_dest[1] = kPad64;
  1940. cur_dest += 2;
  1941. }
  1942. break;
  1943. }
  1944. case 2: {
  1945. // Two bytes left: this encodes to three characters, and (optionally)
  1946. // one pad character to round out the four-character cypherblock.
  1947. if ((szdest -= 3) < 0) return 0;
  1948. uint32 in = BigEndian::Load16(cur_src);
  1949. cur_dest[0] = base64[in >> 10];
  1950. in &= 0x3FF;
  1951. cur_dest[1] = base64[in >> 4];
  1952. in &= 0x00F;
  1953. cur_dest[2] = base64[in << 2];
  1954. cur_dest += 3;
  1955. if (do_padding) {
  1956. if ((szdest -= 1) < 0) return 0;
  1957. cur_dest[0] = kPad64;
  1958. cur_dest += 1;
  1959. }
  1960. break;
  1961. }
  1962. case 3: {
  1963. // Three bytes left: same as in the big loop above. We can't do this in
  1964. // the loop because the loop above always reads 4 bytes, and the fourth
  1965. // byte is past the end of the input.
  1966. if ((szdest -= 4) < 0) return 0;
  1967. uint32 in = (cur_src[0] << 16) + BigEndian::Load16(cur_src + 1);
  1968. cur_dest[0] = base64[in >> 18];
  1969. in &= 0x3FFFF;
  1970. cur_dest[1] = base64[in >> 12];
  1971. in &= 0xFFF;
  1972. cur_dest[2] = base64[in >> 6];
  1973. in &= 0x3F;
  1974. cur_dest[3] = base64[in];
  1975. cur_dest += 4;
  1976. break;
  1977. }
  1978. default:
  1979. // Should not be reached: blocks of 4 bytes are handled
  1980. // in the while loop before this switch statement.
  1981. GOOGLE_LOG(FATAL) << "Logic problem? szsrc = " << szsrc;
  1982. break;
  1983. }
  1984. return (cur_dest - dest);
  1985. }
  1986. static const char kBase64Chars[] =
  1987. "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
  1988. static const char kWebSafeBase64Chars[] =
  1989. "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
  1990. int Base64Escape(const unsigned char *src, int szsrc, char *dest, int szdest) {
  1991. return Base64EscapeInternal(src, szsrc, dest, szdest, kBase64Chars, true);
  1992. }
  1993. int WebSafeBase64Escape(const unsigned char *src, int szsrc, char *dest,
  1994. int szdest, bool do_padding) {
  1995. return Base64EscapeInternal(src, szsrc, dest, szdest,
  1996. kWebSafeBase64Chars, do_padding);
  1997. }
  1998. void Base64EscapeInternal(const unsigned char* src, int szsrc,
  1999. string* dest, bool do_padding,
  2000. const char* base64_chars) {
  2001. const int calc_escaped_size =
  2002. CalculateBase64EscapedLen(szsrc, do_padding);
  2003. dest->resize(calc_escaped_size);
  2004. const int escaped_len = Base64EscapeInternal(src, szsrc,
  2005. string_as_array(dest),
  2006. dest->size(),
  2007. base64_chars,
  2008. do_padding);
  2009. GOOGLE_DCHECK_EQ(calc_escaped_size, escaped_len);
  2010. dest->erase(escaped_len);
  2011. }
  2012. void Base64Escape(const unsigned char *src, int szsrc,
  2013. string* dest, bool do_padding) {
  2014. Base64EscapeInternal(src, szsrc, dest, do_padding, kBase64Chars);
  2015. }
  2016. void WebSafeBase64Escape(const unsigned char *src, int szsrc,
  2017. string *dest, bool do_padding) {
  2018. Base64EscapeInternal(src, szsrc, dest, do_padding, kWebSafeBase64Chars);
  2019. }
  2020. void Base64Escape(StringPiece src, string* dest) {
  2021. Base64Escape(reinterpret_cast<const unsigned char*>(src.data()),
  2022. src.size(), dest, true);
  2023. }
  2024. void WebSafeBase64Escape(StringPiece src, string* dest) {
  2025. WebSafeBase64Escape(reinterpret_cast<const unsigned char*>(src.data()),
  2026. src.size(), dest, false);
  2027. }
  2028. void WebSafeBase64EscapeWithPadding(StringPiece src, string* dest) {
  2029. WebSafeBase64Escape(reinterpret_cast<const unsigned char*>(src.data()),
  2030. src.size(), dest, true);
  2031. }
  2032. // Helper to append a Unicode code point to a string as UTF8, without bringing
  2033. // in any external dependencies.
  2034. int EncodeAsUTF8Char(uint32 code_point, char* output) {
  2035. uint32 tmp = 0;
  2036. int len = 0;
  2037. if (code_point <= 0x7f) {
  2038. tmp = code_point;
  2039. len = 1;
  2040. } else if (code_point <= 0x07ff) {
  2041. tmp = 0x0000c080 |
  2042. ((code_point & 0x07c0) << 2) |
  2043. (code_point & 0x003f);
  2044. len = 2;
  2045. } else if (code_point <= 0xffff) {
  2046. tmp = 0x00e08080 |
  2047. ((code_point & 0xf000) << 4) |
  2048. ((code_point & 0x0fc0) << 2) |
  2049. (code_point & 0x003f);
  2050. len = 3;
  2051. } else {
  2052. // UTF-16 is only defined for code points up to 0x10FFFF, and UTF-8 is
  2053. // normally only defined up to there as well.
  2054. tmp = 0xf0808080 |
  2055. ((code_point & 0x1c0000) << 6) |
  2056. ((code_point & 0x03f000) << 4) |
  2057. ((code_point & 0x000fc0) << 2) |
  2058. (code_point & 0x003f);
  2059. len = 4;
  2060. }
  2061. tmp = ghtonl(tmp);
  2062. memcpy(output, reinterpret_cast<const char*>(&tmp) + sizeof(tmp) - len, len);
  2063. return len;
  2064. }
  2065. // Table of UTF-8 character lengths, based on first byte
  2066. static const unsigned char kUTF8LenTbl[256] = {
  2067. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2068. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2069. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2070. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2071. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2072. 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
  2073. 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
  2074. 3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4
  2075. };
  2076. // Return length of a single UTF-8 source character
  2077. int UTF8FirstLetterNumBytes(const char* src, int len) {
  2078. if (len == 0) {
  2079. return 0;
  2080. }
  2081. return kUTF8LenTbl[*reinterpret_cast<const uint8*>(src)];
  2082. }
  2083. // ----------------------------------------------------------------------
  2084. // CleanStringLineEndings()
  2085. // Clean up a multi-line string to conform to Unix line endings.
  2086. // Reads from src and appends to dst, so usually dst should be empty.
  2087. //
  2088. // If there is no line ending at the end of a non-empty string, it can
  2089. // be added automatically.
  2090. //
  2091. // Four different types of input are correctly handled:
  2092. //
  2093. // - Unix/Linux files: line ending is LF: pass through unchanged
  2094. //
  2095. // - DOS/Windows files: line ending is CRLF: convert to LF
  2096. //
  2097. // - Legacy Mac files: line ending is CR: convert to LF
  2098. //
  2099. // - Garbled files: random line endings: convert gracefully
  2100. // lonely CR, lonely LF, CRLF: convert to LF
  2101. //
  2102. // @param src The multi-line string to convert
  2103. // @param dst The converted string is appended to this string
  2104. // @param auto_end_last_line Automatically terminate the last line
  2105. //
  2106. // Limitations:
  2107. //
  2108. // This does not do the right thing for CRCRLF files created by
  2109. // broken programs that do another Unix->DOS conversion on files
  2110. // that are already in CRLF format. For this, a two-pass approach
  2111. // brute-force would be needed that
  2112. //
  2113. // (1) determines the presence of LF (first one is ok)
  2114. // (2) if yes, removes any CR, else convert every CR to LF
  2115. void CleanStringLineEndings(const string &src, string *dst,
  2116. bool auto_end_last_line) {
  2117. if (dst->empty()) {
  2118. dst->append(src);
  2119. CleanStringLineEndings(dst, auto_end_last_line);
  2120. } else {
  2121. string tmp = src;
  2122. CleanStringLineEndings(&tmp, auto_end_last_line);
  2123. dst->append(tmp);
  2124. }
  2125. }
  2126. void CleanStringLineEndings(string *str, bool auto_end_last_line) {
  2127. ptrdiff_t output_pos = 0;
  2128. bool r_seen = false;
  2129. ptrdiff_t len = str->size();
  2130. char *p = &(*str)[0];
  2131. for (ptrdiff_t input_pos = 0; input_pos < len;) {
  2132. if (!r_seen && input_pos + 8 < len) {
  2133. uint64_t v = GOOGLE_UNALIGNED_LOAD64(p + input_pos);
  2134. // Loop over groups of 8 bytes at a time until we come across
  2135. // a word that has a byte whose value is less than or equal to
  2136. // '\r' (i.e. could contain a \n (0x0a) or a \r (0x0d) ).
  2137. //
  2138. // We use a has_less macro that quickly tests a whole 64-bit
  2139. // word to see if any of the bytes has a value < N.
  2140. //
  2141. // For more details, see:
  2142. // http://graphics.stanford.edu/~seander/bithacks.html#HasLessInWord
  2143. #define has_less(x, n) (((x) - ~0ULL / 255 * (n)) & ~(x) & ~0ULL / 255 * 128)
  2144. if (!has_less(v, '\r' + 1)) {
  2145. #undef has_less
  2146. // No byte in this word has a value that could be a \r or a \n
  2147. if (output_pos != input_pos) {
  2148. GOOGLE_UNALIGNED_STORE64(p + output_pos, v);
  2149. }
  2150. input_pos += 8;
  2151. output_pos += 8;
  2152. continue;
  2153. }
  2154. }
  2155. string::const_reference in = p[input_pos];
  2156. if (in == '\r') {
  2157. if (r_seen) p[output_pos++] = '\n';
  2158. r_seen = true;
  2159. } else if (in == '\n') {
  2160. if (input_pos != output_pos)
  2161. p[output_pos++] = '\n';
  2162. else
  2163. output_pos++;
  2164. r_seen = false;
  2165. } else {
  2166. if (r_seen) p[output_pos++] = '\n';
  2167. r_seen = false;
  2168. if (input_pos != output_pos)
  2169. p[output_pos++] = in;
  2170. else
  2171. output_pos++;
  2172. }
  2173. input_pos++;
  2174. }
  2175. if (r_seen ||
  2176. (auto_end_last_line && output_pos > 0 && p[output_pos - 1] != '\n')) {
  2177. str->resize(output_pos + 1);
  2178. str->operator[](output_pos) = '\n';
  2179. } else if (output_pos < len) {
  2180. str->resize(output_pos);
  2181. }
  2182. }
  2183. } // namespace protobuf
  2184. } // namespace google