Rewrite `WideToUtf8` for improved readability. This is supposed to be a zero-diff change. PiperOrigin-RevId: 756859112 Change-Id: Ia81a84bc5d1e6f2a1299ca0ff5dbcec48583ab76
diff --git a/absl/strings/internal/utf8.cc b/absl/strings/internal/utf8.cc index 4370c7c..61945f5 100644 --- a/absl/strings/internal/utf8.cc +++ b/absl/strings/internal/utf8.cc
@@ -18,6 +18,7 @@ #include <cstddef> #include <cstdint> +#include <limits> #include "absl/base/config.h" @@ -25,7 +26,7 @@ ABSL_NAMESPACE_BEGIN namespace strings_internal { -size_t EncodeUTF8Char(char *buffer, char32_t utf8_char) { +size_t EncodeUTF8Char(char* buffer, char32_t utf8_char) { if (utf8_char <= 0x7F) { *buffer = static_cast<char>(utf8_char); return 1; @@ -53,45 +54,93 @@ } } -size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s) { - const auto v = static_cast<uint32_t>(wc); - if (v < 0x80) { - *buf = static_cast<char>(v); +size_t WideToUtf8(wchar_t wc, char* buf, ShiftState& s) { + // Reinterpret the output buffer `buf` as `unsigned char*` for subsequent + // bitwise operations. This ensures well-defined behavior for bit + // manipulations (avoiding issues with signed `char`) and is safe under C++ + // aliasing rules, as `unsigned char` can alias any type. + auto* ubuf = reinterpret_cast<unsigned char*>(buf); + const uint32_t v = static_cast<uint32_t>(wc); + constexpr size_t kError = static_cast<size_t>(-1); + + if (v <= 0x007F) { + // 1-byte sequence (U+0000 to U+007F). + // 0xxxxxxx. + ubuf[0] = (0b0111'1111 & v); + s = {}; // Reset surrogate state. return 1; - } else if (v < 0x800) { - *buf++ = static_cast<char>(0xc0 | (v >> 6)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); + } else if (0x0080 <= v && v <= 0x07FF) { + // 2-byte sequence (U+0080 to U+07FF). + // 110xxxxx 10xxxxxx. + ubuf[0] = 0b1100'0000 | (0b0001'1111 & (v >> 6)); + ubuf[1] = 0b1000'0000 | (0b0011'1111 & v); + s = {}; // Reset surrogate state. return 2; - } else if (v < 0xd800 || (v - 0xe000) < 0x2000) { - *buf++ = static_cast<char>(0xe0 | (v >> 12)); - *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); + } else if ((0x0800 <= v && v <= 0xD7FF) || (0xE000 <= v && v <= 0xFFFF)) { + // 3-byte sequence (U+0800 to U+D7FF or U+E000 to U+FFFF). + // Excludes surrogate code points U+D800-U+DFFF. + // 1110xxxx 10xxxxxx 10xxxxxx. + ubuf[0] = 0b1110'0000 | (0b0000'1111 & (v >> 12)); + ubuf[1] = 0b1000'0000 | (0b0011'1111 & (v >> 6)); + ubuf[2] = 0b1000'0000 | (0b0011'1111 & v); + s = {}; // Reset surrogate state. return 3; - } else if ((v - 0x10000) < 0x100000) { - *buf++ = static_cast<char>(0xf0 | (v >> 18)); - *buf++ = static_cast<char>(0x80 | ((v >> 12) & 0x3f)); - *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - return 4; - } else if (v < 0xdc00) { - s.saw_high_surrogate = true; - s.bits = static_cast<uint8_t>(v & 0x3); - const uint8_t high_bits = ((v >> 6) & 0xf) + 1; - *buf++ = static_cast<char>(0xf0 | (high_bits >> 2)); - *buf = - static_cast<char>(0x80 | static_cast<uint8_t>((high_bits & 0x3) << 4) | - static_cast<uint8_t>((v >> 2) & 0xf)); - return 2; - } else if (v < 0xe000 && s.saw_high_surrogate) { - *buf++ = static_cast<char>(0x80 | static_cast<uint8_t>(s.bits << 4) | - static_cast<uint8_t>((v >> 6) & 0xf)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - s.saw_high_surrogate = false; - s.bits = 0; - return 2; - } else { - return static_cast<size_t>(-1); + } else if (0xD800 <= v && v <= 0xDBFF) { + // High Surrogate (U+D800 to U+DBFF). + // This part forms the first two bytes of an eventual 4-byte UTF-8 sequence. + const unsigned char high_bits_val = (0b0000'1111 & (v >> 6)) + 1; + + // First byte of the 4-byte UTF-8 sequence (11110xxx). + ubuf[0] = 0b1111'0000 | (0b0000'0111 & (high_bits_val >> 2)); + // Second byte of the 4-byte UTF-8 sequence (10xxxxxx). + ubuf[1] = 0b1000'0000 | // + (0b0011'0000 & (high_bits_val << 4)) | // + (0b0000'1111 & (v >> 2)); + // Set state for high surrogate after writing to buffer. + s = {true, static_cast<unsigned char>(0b0000'0011 & v)}; + return 2; // Wrote 2 bytes, expecting 2 more from a low surrogate. + } else if (0xDC00 <= v && v <= 0xDFFF) { + // Low Surrogate (U+DC00 to U+DFFF). + // This part forms the last two bytes of a 4-byte UTF-8 sequence, + // using state from a preceding high surrogate. + if (!s.saw_high_surrogate) { + // Error: Isolated low surrogate without a preceding high surrogate. + // s remains in its current (problematic) state. + // Caller should handle error. + return kError; + } + + // Third byte of the 4-byte UTF-8 sequence (10xxxxxx). + ubuf[0] = 0b1000'0000 | // + (0b0011'0000 & (s.bits << 4)) | // + (0b0000'1111 & (v >> 6)); + // Fourth byte of the 4-byte UTF-8 sequence (10xxxxxx). + ubuf[1] = 0b1000'0000 | (0b0011'1111 & v); + + s = {}; // Reset surrogate state, pair complete. + return 2; // Wrote 2 more bytes, completing the 4-byte sequence. + } else if constexpr (0xFFFF < std::numeric_limits<wchar_t>::max()) { + // Conditionally compile the 4-byte direct conversion branch. + // This block is compiled only if wchar_t can represent values > 0xFFFF. + // It's placed after surrogate checks to ensure surrogates are handled by + // their specific logic. This inner 'if' is the runtime check for the 4-byte + // range. At this point, v is known not to be in the 1, 2, or 3-byte BMP + // ranges, nor is it a surrogate code point. + if (0x10000 <= v && v <= 0x10FFFF) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx. + ubuf[0] = 0b1111'0000 | (0b0000'0111 & (v >> 18)); + ubuf[1] = 0b1000'0000 | (0b0011'1111 & (v >> 12)); + ubuf[2] = 0b1000'0000 | (0b0011'1111 & (v >> 6)); + ubuf[3] = 0b1000'0000 | (0b0011'1111 & v); + s = {}; // Reset surrogate state. + return 4; + } } + + // Invalid wchar_t value (e.g., out of Unicode range, or unhandled after all + // checks). + s = {}; // Reset surrogate state. + return kError; } } // namespace strings_internal
diff --git a/absl/strings/internal/utf8.h b/absl/strings/internal/utf8.h index f240408..ed1db11 100644 --- a/absl/strings/internal/utf8.h +++ b/absl/strings/internal/utf8.h
@@ -41,11 +41,11 @@ // characters into buffer, however never will more than kMaxEncodedUTF8Size // bytes be written, regardless of the value of utf8_char. enum { kMaxEncodedUTF8Size = 4 }; -size_t EncodeUTF8Char(char *buffer, char32_t utf8_char); +size_t EncodeUTF8Char(char* buffer, char32_t utf8_char); struct ShiftState { bool saw_high_surrogate = false; - uint8_t bits = 0; + unsigned char bits = 0; }; // Converts `wc` from UTF-16 or UTF-32 to UTF-8 and writes to `buf`. `buf` is @@ -55,7 +55,7 @@ // // This is basically std::wcrtomb(), but always outputting UTF-8 instead of // respecting the current locale. -size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s); +size_t WideToUtf8(wchar_t wc, char* buf, ShiftState& s); } // namespace strings_internal ABSL_NAMESPACE_END
diff --git a/absl/strings/internal/utf8_test.cc b/absl/strings/internal/utf8_test.cc index 62322dd..b88d7bb 100644 --- a/absl/strings/internal/utf8_test.cc +++ b/absl/strings/internal/utf8_test.cc
@@ -103,8 +103,21 @@ {"BMP_MaxBeforeSurrogates_D7FF", L'\uD7FF', "\xED\x9F\xBF", 3}, {"BMP_FFFF", L'\uFFFF', "\xEF\xBF\xBF", 3}, - {"IsolatedHighSurr_D800", L'\xD800', "\xF0\x90", 2, {true, 0}, {true, 0}}, - {"IsolatedHighSurr_DBFF", L'\xDBFF', "\xF4\x8F", 2, {true, 3}, {true, 3}}, + {"IsolatedHighSurr_D800", L'\xD800', "\xF0\x90", 2, {}, {true, 0}}, + {"IsolatedHighSurr_DBFF", L'\xDBFF', "\xF4\x8F", 2, {}, {true, 3}}, + + {"HighSurr_D800_after_HighD800", + L'\xD800', + "\xF0\x90", + 2, + {true, 0}, + {true, 0}}, + {"HighSurr_DBFF_after_HighDBFF", + L'\xDBFF', + "\xF4\x8F", + 2, + {true, 3}, + {true, 3}}, {"LowSurr_DC00_after_HighD800", L'\xDC00', "\x80\x80", 2, {true, 0}, {}}, {"LowSurr_DFFD_after_HighDBFF", L'\xDFFD', "\xBF\xBD", 2, {true, 3}, {}},