Have std/json reject some invalid UTF-8
diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 73f4581..9957b36 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c
@@ -5290,6 +5290,9 @@ // ---------------- Status Codes +extern const char* wuffs_json__error__bad_c0_control_code; +extern const char* wuffs_json__error__bad_utf_8; +extern const char* wuffs_json__error__bad_backslash_escape; extern const char* wuffs_json__error__bad_input; extern const char* wuffs_json__error__unsupported_number_length; extern const char* wuffs_json__error__unsupported_recursion_depth; @@ -18717,6 +18720,11 @@ // ---------------- Status Codes Implementations +const char* wuffs_json__error__bad_c0_control_code = + "#json: bad C0 control code"; +const char* wuffs_json__error__bad_utf_8 = "#json: bad UTF-8"; +const char* wuffs_json__error__bad_backslash_escape = + "#json: bad backslash-escape"; const char* wuffs_json__error__bad_input = "#json: bad input"; const char* wuffs_json__error__unsupported_number_length = "#json: unsupported number length"; @@ -18746,6 +18754,30 @@ 0, 0, 0, 0, 0, 0, 0, 0, 0, }; +static const uint8_t // + wuffs_json__lut_chars[256] // + WUFFS_BASE__POTENTIALLY_UNUSED = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 129, 129, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 5, 5, 5, 5, 5, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 129, 129, +}; + static const uint8_t // wuffs_json__lut_classes[256] // WUFFS_BASE__POTENTIALLY_UNUSED = { @@ -18905,8 +18937,9 @@ uint32_t v_stack_bit = 0; uint32_t v_match = 0; uint8_t v_c = 0; - uint8_t v_class = 0; uint8_t v_backslash = 0; + uint8_t v_char = 0; + uint8_t v_class = 0; uint8_t v_uni4_ok = 0; uint64_t v_uni4_string = 0; uint32_t v_uni4_value = 0; @@ -19051,6 +19084,7 @@ goto label__string_loop__continue; } v_string_length = 0; + label__1__continue:; while (true) { if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_string_length > 0) { @@ -19073,29 +19107,28 @@ goto label__string_loop__continue; } v_c = wuffs_base__load_u8be(iop_a_src); - if (v_c <= 34) { - if (v_c == 34) { + v_char = wuffs_json__lut_chars[v_c]; + if (v_char == 0) { + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if (v_string_length >= 65534) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097696)) + (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(((uint64_t)(v_string_length)))) + (((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; - goto label__string_loop__break; - } else if (v_c < 32) { - if (v_string_length > 0) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097697)) - << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(((uint64_t)(v_string_length)))) - << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_string_length = 0; - } - status = - wuffs_base__make_status(wuffs_json__error__bad_input); - goto exit; + goto label__string_loop__continue; } - } else if (v_c == 92) { + v_string_length += 1; + goto label__1__continue; + } else if (v_char == 1) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097696)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)(v_string_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop__break; + } else if (v_char == 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2097697)) @@ -19109,37 +19142,39 @@ } if (((uint64_t)(io2_a_src - iop_a_src)) < 2) { if (a_src && a_src->meta.closed) { - status = - wuffs_base__make_status(wuffs_json__error__bad_input); + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); goto exit; } status = wuffs_base__make_status( wuffs_base__suspension__short_read); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7); v_string_length = 0; + v_char = 0; goto label__string_loop__continue; } - v_backslash = - ((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8))); - v_c = wuffs_json__lut_backslashes[v_backslash]; - if (v_c > 0) { + v_c = ((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8))); + v_backslash = wuffs_json__lut_backslashes[v_c]; + if (v_backslash > 0) { (iop_a_src += 2, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((4194304 | ((uint64_t)((v_c & 127)))))) + (((uint64_t)( + (4194304 | ((uint64_t)((v_backslash & 127)))))) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop__continue; - } else if (v_backslash == 117) { + } else if (v_c == 117) { if (((uint64_t)(io2_a_src - iop_a_src)) < 6) { if (a_src && a_src->meta.closed) { - status = - wuffs_base__make_status(wuffs_json__error__bad_input); + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); goto exit; } status = wuffs_base__make_status( wuffs_base__suspension__short_read); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8); v_string_length = 0; + v_char = 0; goto label__string_loop__continue; } v_uni4_string = @@ -19175,7 +19210,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) < 12) { if (a_src && a_src->meta.closed) { status = wuffs_base__make_status( - wuffs_json__error__bad_input); + wuffs_json__error__bad_backslash_escape); goto exit; } status = wuffs_base__make_status( @@ -19183,6 +19218,7 @@ WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); v_string_length = 0; v_uni4_value = 0; + v_char = 0; goto label__string_loop__continue; } (iop_a_src += 4, wuffs_base__make_empty_struct()); @@ -19242,22 +19278,41 @@ } } } - status = wuffs_base__make_status(wuffs_json__error__bad_input); + status = wuffs_base__make_status( + wuffs_json__error__bad_backslash_escape); goto exit; + } else if (v_char <= 16) { + (iop_a_src += 1, wuffs_base__make_empty_struct()); + if (v_string_length >= 65534) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(2097697)) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(65535)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop__continue; + } + v_string_length += 1; + goto label__1__continue; } - (iop_a_src += 1, wuffs_base__make_empty_struct()); - if (v_string_length >= 65534) { + if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (((uint64_t)(((uint64_t)(v_string_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; - goto label__string_loop__continue; } - v_string_length += 1; + if (v_char == 128) { + status = wuffs_base__make_status( + wuffs_json__error__bad_c0_control_code); + goto exit; + } + status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); + goto exit; } } label__string_loop__break:; - label__1__continue:; + label__2__continue:; while (true) { if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (a_src && a_src->meta.closed) { @@ -19267,21 +19322,21 @@ status = wuffs_base__make_status(wuffs_base__suspension__short_read); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); - goto label__1__continue; + goto label__2__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); - goto label__1__continue; + goto label__2__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label__1__break; + goto label__2__break; } - label__1__break:; + label__2__break:; if (0 == (v_expect & 16)) { v_expect = 8; goto label__outer__continue; @@ -19326,7 +19381,7 @@ << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_number_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label__2__break; + goto label__3__break; } while (v_number_length > 0) { v_number_length -= 1; @@ -19356,7 +19411,7 @@ } } } - label__2__break:; + label__3__break:; goto label__goto_parsed_a_leaf_value__break; } else if (v_class == 5) { v_token_value = 8405009;
diff --git a/std/json/common_consts.wuffs b/std/json/common_consts.wuffs index 8544a49..385cf26 100644 --- a/std/json/common_consts.wuffs +++ b/std/json/common_consts.wuffs
@@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub status "#bad C0 control code" +pub status "#bad UTF-8" +pub status "#bad backslash-escape" pub status "#bad input" pub status "#unsupported number length" pub status "#unsupported recursion depth" @@ -62,6 +65,68 @@ // 8 9 A B C D E F ] +// lut_chars helps decode bytes within a string: +// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code. +// - 0x01 is '"'. +// - 0x02 is '\\'. +// - 0x03 is the start of 2-byte UTF-8. +// - 0x04 is the start of 3-byte UTF-8. +// - 0x05 is the start of 4-byte UTF-8. +// - 0x10 is a UTF-8 tail byte. +// - 0x80 is invalid JSON (C0 control codes). +// - 0x81 is invalid UTF-8. +// +// RFC 3629 (UTF-8) gives this grammar for valid UTF-8: +// UTF8-1 = %x00-7F +// UTF8-2 = %xC2-DF UTF8-tail +// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / +// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) +// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / +// %xF4 %x80-8F 2( UTF8-tail ) +// UTF8-tail = %x80-BF +pri const lut_chars array[256] base.u8 = [ + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes. + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes. + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes. + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes. + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. UTF-8-1. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. UTF-8-1; '\\'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. UTF-8-1. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. UTF-8-1. + + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x80 ..= 0x87. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x88 ..= 0x8F. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x90 ..= 0x97. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x98 ..= 0x9F. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA0 ..= 0xA7. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA8 ..= 0xAF. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail. + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail. + + 0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2. + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2. + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2. + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2. + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3. + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3. + 0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8. + 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8. + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F +] + +// lut_classes is: // - 0x00 (bitmask 0x0001) is CLASS_WHITESPACE. // - 0x01 (bitmask 0x0002) is CLASS_STRING. // - 0x02 (bitmask 0x0004) is CLASS_COMMA.
diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index d4db482..64746cf 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs
@@ -55,8 +55,9 @@ var stack_bit : base.u32[..= 31] var match : base.u32[..= 2] var c : base.u8 - var class : base.u8[..= 0x0F] var backslash : base.u8 + var char : base.u8 + var class : base.u8[..= 0x0F] var uni4_ok : base.u8 var uni4_string : base.u64 @@ -203,26 +204,28 @@ continue.string_loop } c = args.src.peek_u8() + char = lut_chars[c] - if c <= 0x22 { // 0x22 is '"'. - if c == 0x22 { // 0x22 is '"'. + if char == 0x00 { // Non-special ASCII. + args.src.skip32_fast!(actual: 1, worst_case: 1) + if string_length >= 0xFFFE { args.dst.write_fast_token!( - value: 0x20_0220, - length: string_length as base.u64) + value: 0x20_0221, + length: 0xFFFF) string_length = 0 - break.string_loop - - } else if c < 0x20 { // 0x20 is ' '. - if string_length > 0 { - args.dst.write_fast_token!( - value: 0x20_0221, - length: string_length as base.u64) - string_length = 0 - } - return "#bad input" + continue.string_loop } + string_length += 1 + continue - } else if c == 0x5C { // 0x5C is '\\'. + } else if char == 0x01 { // '"' + args.dst.write_fast_token!( + value: 0x20_0220, + length: string_length as base.u64) + string_length = 0 + break.string_loop + + } else if char == 0x02 { // '\\'. if string_length > 0 { args.dst.write_fast_token!( value: 0x20_0221, @@ -236,29 +239,31 @@ if args.src.available() < 2 { if args.src.is_closed() { - return "#bad input" + return "#bad backslash-escape" } yield? base."$short read" string_length = 0 + char = 0 continue.string_loop } - backslash = (args.src.peek_u16le() >> 8) as base.u8 - c = lut_backslashes[backslash] - if c > 0 { + c = (args.src.peek_u16le() >> 8) as base.u8 + backslash = lut_backslashes[c] + if backslash > 0 { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( - value: 0x40_0000 | ((c & 0x7F) as base.u64), + value: 0x40_0000 | ((backslash & 0x7F) as base.u64), length: 2) continue.string_loop - } else if backslash == 0x75 { // 0x75 is 'u'. + } else if c == 0x75 { // 0x75 is 'u'. // -------- BEGIN backslash-u. if args.src.available() < 6 { if args.src.is_closed() { - return "#bad input" + return "#bad backslash-escape" } yield? base."$short read" string_length = 0 + char = 0 continue.string_loop } @@ -281,7 +286,8 @@ if uni4_ok == 0 { // It wasn't 4 hexadecimal digits. No-op - // (and fall through to "#bad input"). + // (and fall through to "#bad + // backslash-escape"). } else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) { // Not a Unicode surrogate. We're good. @@ -293,7 +299,7 @@ } else if uni4_value >= 0xDC00 { // Low surrogate. No-op (and fall through - // to "#bad input"). + // to "#bad backslash-escape"). } else { // High surrogate, which needs to be @@ -302,11 +308,12 @@ // high surrogate. We need another 6. if args.src.available() < 12 { if args.src.is_closed() { - return "#bad input" + return "#bad backslash-escape" } yield? base."$short read" string_length = 0 uni4_value = 0 + char = 0 continue.string_loop } @@ -356,7 +363,8 @@ } else { // Roll back the 4 bytes, then fall - // through to "#bad input". + // through to "#invalid + // backslash-escape". uni4_rollback = 4 while uni4_rollback > 0 { uni4_rollback -= 1 @@ -371,18 +379,32 @@ // -------- END backslash-u. } - return "#bad input" + return "#bad backslash-escape" + + } else if char <= 0x10 { + // TODO: reject invalid UTF-8. + args.src.skip32_fast!(actual: 1, worst_case: 1) + if string_length >= 0xFFFE { + args.dst.write_fast_token!( + value: 0x20_0221, + length: 0xFFFF) + string_length = 0 + continue.string_loop + } + string_length += 1 + continue } - args.src.skip32_fast!(actual: 1, worst_case: 1) - if string_length >= 0xFFFE { + if string_length > 0 { args.dst.write_fast_token!( value: 0x20_0221, - length: 0xFFFF) + length: string_length as base.u64) string_length = 0 - continue.string_loop } - string_length += 1 + if char == 0x80 { + return "#bad C0 control code" + } + return "#bad UTF-8" } } endwhile.string_loop
diff --git a/test/c/std/json.c b/test/c/std/json.c index c5b9d58..8e52262 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c
@@ -219,7 +219,7 @@ // ---------------- JSON Tests const char* // -test_wuffs_json_decode_tokens() { +test_wuffs_json_decode_interface() { CHECK_FOCUS(__func__); wuffs_json__decoder dec; CHECK_STATUS("initialize", @@ -375,7 +375,88 @@ } if (total_length != src.meta.ri) { - RETURN_FAIL("%s: total length: have 0x%" PRIu64 ", want 0x%" PRIu64, + RETURN_FAIL("%s: total length: have %" PRIu64 ", want %" PRIu64, + test_cases[tc].str, total_length, src.meta.ri); + } + } + + return NULL; +} + +const char* // +test_wuffs_json_decode_string() { + CHECK_FOCUS(__func__); + + const char* bad_bac = wuffs_json__error__bad_backslash_escape; + const char* bad_ccc = wuffs_json__error__bad_c0_control_code; + const char* bad_utf = wuffs_json__error__bad_utf_8; + + struct { + const char* want_status_repr; + const char* str; + } test_cases[] = { + {.want_status_repr = NULL, .str = "\"+++\\\"+\\/+\\\\+++\""}, + {.want_status_repr = NULL, .str = "\"+++\\b+\\f+\\n+\\r+\\t+++\""}, + {.want_status_repr = NULL, .str = "\"\x20\""}, // U+00000020. + {.want_status_repr = NULL, .str = "\"\xC2\x80\""}, // U+00000080. + {.want_status_repr = NULL, .str = "\"\xDF\xBF\""}, // U+000007FF. + {.want_status_repr = NULL, .str = "\"\xE0\xA0\x80\""}, // U+00000800. + {.want_status_repr = NULL, .str = "\"\xED\xAF\xBF\""}, // U+0000DBFF. + {.want_status_repr = NULL, .str = "\"\xEE\x80\x80\""}, // U+0000E000. + {.want_status_repr = NULL, .str = "\"\xEF\xBF\xBF\""}, // U+0000FFFF. + {.want_status_repr = NULL, .str = "\"\xF0\x90\x80\x80\""}, // U+00010000. + {.want_status_repr = NULL, .str = "\"\xF4\x8F\xBF\xBF\""}, // U+0010FFFF. + {.want_status_repr = NULL, .str = "\"abc\""}, + {.want_status_repr = NULL, .str = "\"i\x6Ak\""}, + {.want_status_repr = NULL, .str = "\"space+\x20+space\""}, + {.want_status_repr = NULL, .str = "\"tab+\\t+tab\""}, + {.want_status_repr = NULL, .str = "\"tab+\\u0009+tab\""}, + + {.want_status_repr = bad_bac, .str = "\"\\uIJKL\""}, + {.want_status_repr = bad_bac, .str = "\"space+\\x20+space\""}, + + {.want_status_repr = bad_ccc, .str = "\"\x1F\""}, + {.want_status_repr = bad_ccc, .str = "\"tab+\t+tab\""}, + + {.want_status_repr = bad_utf, .str = "\"\xF5\""}, + {.want_status_repr = bad_utf, .str = "\"\xFF\xFF\xFF\xFF\""}, + }; + + wuffs_json__decoder dec; + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + CHECK_STATUS("initialize", + wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = global_have_token_slice, + }); + size_t n = strlen(test_cases[tc].str); + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = wuffs_base__make_slice_u8((void*)(test_cases[tc].str), n), + .meta = wuffs_base__make_io_buffer_meta(n, 0, 0, true), + }); + + wuffs_base__status have_status = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src); + + uint64_t total_length = 0; + size_t i; + for (i = tok.meta.ri; i < tok.meta.wi; i++) { + wuffs_base__token* t = &tok.data.ptr[i]; + total_length = + wuffs_base__u64__sat_add(total_length, wuffs_base__token__length(t)); + } + + if (have_status.repr != test_cases[tc].want_status_repr) { + RETURN_FAIL("%s: have \"%s\", want \"%s\"", test_cases[tc].str, + have_status.repr, test_cases[tc].want_status_repr); + } + + if (total_length != src.meta.ri) { + RETURN_FAIL("%s: total length: have %" PRIu64 ", want %" PRIu64, test_cases[tc].str, total_length, src.meta.ri); } } @@ -444,7 +525,8 @@ test_strconv_parse_number_i64, // test_strconv_parse_number_u64, // - test_wuffs_json_decode_tokens, // + test_wuffs_json_decode_interface, // + test_wuffs_json_decode_string, // test_wuffs_json_decode_unicode4_escapes, // #ifdef WUFFS_MIMIC