| // Copyright 2020 The Wuffs Authors. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // https://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| pub struct decoder? implements base.token_decoder( | |
| quirks : array[QUIRKS_COUNT] base.bool, | |
| allow_leading_ars : base.bool, | |
| allow_leading_ubom : base.bool, | |
| end_of_data : base.bool, | |
| trailer_stop : base.u8, | |
| // comment_type is set as a side-effect of decode_comment?. | |
| // - 0 means no comment. | |
| // - 1 means a block comment. | |
| // - 2 means a line comment. | |
| comment_type : base.u8, | |
| util : base.utility, | |
| ) + ( | |
| // stack is conceptually an array of bits, implemented as an array of u32. | |
| // The N'th bit being 0 or 1 means that we're in an array or object, where | |
| // N is the recursion depth. | |
| // | |
| // Parsing JSON involves recursion: containers (arrays and objects) can | |
| // hold other containers. As child elements are completed, the parser needs | |
| // to remember 1 bit of state per recursion depth: whether the parent | |
| // container was an array or an object. When continuing to parse the | |
| // parent's elements, `, "key": value` is only valid for objects. | |
| // | |
| // Note that we explicitly track our own stack and depth. We do not use the | |
| // call stack to hold this state and the decoder.decode_tokens function is | |
| // not recursive per se. | |
| // | |
| // Wuffs code does not have the capability to dynamically allocate memory, | |
| // so the maximum depth is hard-coded at compile time. In this case, the | |
| // maximum is 1024 (stack is 1024 bits or 128 bytes), also known as | |
| // DECODER_DEPTH_MAX_INCL. | |
| // | |
| // The [JSON spec](https://www.ietf.org/rfc/rfc8259.txt) clearly states, | |
| // "an implementation may set limits on the maximum depth of nesting". | |
| // | |
| // In comparison, as of February 2020, the Chromium web browser's JSON | |
| // parser's maximum recursion depth is 200: | |
| // https://source.chromium.org/chromium/chromium/src/+/3dece34cde622faa0daac07156c25d92c9897d1e:base/json/json_common.h;l=18 | |
| // | |
| // Other languages and libraries' maximum depths (determined empirically) | |
| // are listed at https://github.com/lovasoa/bad_json_parsers#results | |
| stack : array[1024 / 32] base.u32, | |
| ) | |
| pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status { | |
| if args.key >= QUIRKS_BASE { | |
| args.key -= QUIRKS_BASE | |
| if args.key < QUIRKS_COUNT { | |
| this.quirks[args.key] = args.value > 0 | |
| return ok | |
| } | |
| } | |
| return base."#unsupported option" | |
| } | |
| pub func decoder.workbuf_len() base.range_ii_u64 { | |
| return this.util.empty_range_ii_u64() | |
| } | |
| pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, workbuf: slice base.u8) { | |
| // This is a very, very long function, and it is tempting to refactor it. | |
| // Be careful of performance impacts when doing so. For example, commit | |
| // 86d3b89f "Factor out json.decoder.decode_string" pulled out a 500 line | |
| // decode_string function, which was certainly cleaner structurally, but | |
| // also regressed performance by 1.1x to 1.2x. For details, see | |
| // https://github.com/google/wuffs/commit/86d3b89f9a6578d964a4b6d71e21dfc9bb702b44 | |
| var vminor : base.u32[..= 0xFF_FFFF] | |
| var number_length : base.u32[..= 0x3FF] | |
| var number_status : base.u32[..= 0x3] | |
| var string_length : base.u32[..= 0xFFFB] | |
| var whitespace_length : base.u32[..= 0xFFFE] | |
| var depth : base.u32[..= 1024] | |
| var stack_byte : base.u32[..= (1024 / 32) - 1] | |
| var stack_bit : base.u32[..= 31] | |
| var match : base.u32[..= 2] | |
| var c4 : base.u32 | |
| var c : base.u8 | |
| var backslash : base.u8 | |
| var char : base.u8 | |
| var class : base.u8[..= 0x0F] | |
| var multi_byte_utf8 : base.u32 | |
| var backslash_x_ok : base.u8 | |
| var backslash_x_value : base.u8 | |
| var backslash_x_string : base.u32 | |
| var uni4_ok : base.u8 | |
| var uni4_string : base.u64 | |
| var uni4_value : base.u32[..= 0xFFFF] | |
| var uni4_high_surrogate : base.u32[..= 0x10_FC00] | |
| var uni8_ok : base.u8 | |
| var uni8_string : base.u64 | |
| var uni8_value : base.u32[..= 0xFFFF_FFFF] | |
| // expect is a bitmask of what the next character class can be. | |
| // | |
| // expect_after_value is what to expect after seeing a value (a literal, | |
| // number, string, array or object). For depth 0, this is ignored. | |
| // Otherwise, it should be (EXPECT_CLOSE_FOO | EXPECT_COMMA), for some | |
| // value of FOO. | |
| var expect : base.u32 | |
| var expect_after_value : base.u32 | |
| if this.end_of_data { | |
| return base."@end of data" | |
| } | |
| if this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] { | |
| if this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] or | |
| this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] or | |
| this.quirks[QUIRK_ALLOW_TRAILING_FILLER - QUIRKS_BASE] { | |
| return "#bad quirk combination" | |
| } | |
| } | |
| if this.quirks[QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR - QUIRKS_BASE] or | |
| this.quirks[QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK - QUIRKS_BASE] { | |
| this.decode_leading?(dst: args.dst, src: args.src) | |
| } | |
| expect = EXPECT_VALUE | |
| while.outer true { | |
| while.goto_parsed_a_leaf_value true {{ | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue.outer | |
| } | |
| // Consume whitespace. | |
| whitespace_length = 0 | |
| c = 0 | |
| class = 0 | |
| while.ws true, | |
| inv args.dst.length() > 0, | |
| post args.src.length() > 0, | |
| { | |
| if args.src.length() <= 0 { | |
| if whitespace_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: 0, | |
| continued: 0, | |
| length: whitespace_length) | |
| whitespace_length = 0 | |
| } | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| continue.outer | |
| } | |
| c = args.src.peek_u8() | |
| class = LUT_CLASSES[c] | |
| if class <> CLASS_WHITESPACE { | |
| break.ws | |
| } | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if whitespace_length >= 0xFFFE { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: 0, | |
| continued: 0, | |
| length: 0xFFFF) | |
| whitespace_length = 0 | |
| continue.outer | |
| } | |
| whitespace_length += 1 | |
| } endwhile.ws | |
| // Emit whitespace. | |
| if whitespace_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: 0, | |
| continued: 0, | |
| length: whitespace_length) | |
| whitespace_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.outer | |
| } | |
| } | |
| // Check expected character classes. | |
| if 0 == (expect & ((1 as base.u32) << class)) { | |
| return "#bad input" | |
| } | |
| // These assertions are redundant (the Wuffs compiler should already | |
| // know these facts; deleting these assertions should still compile) | |
| // but are listed explicitly to guard against future edits to the code | |
| // above inadvertently invalidating these assertions. | |
| assert args.dst.length() > 0 | |
| assert args.src.length() > 0 | |
| if class == CLASS_STRING { | |
| // -------- BEGIN parse strings. | |
| // Emit the leading '"'. | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__DEFINITELY_ASCII | | |
| base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| while.string_loop_outer true { | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue.string_loop_outer | |
| } | |
| string_length = 0 | |
| while.string_loop_inner true, | |
| pre args.dst.length() > 0, | |
| { | |
| if args.src.length() <= 0 { | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| } | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| // As an optimization, consume non-special ASCII 4 bytes at | |
| // a time. | |
| while args.src.length() > 4, | |
| inv args.dst.length() > 0, | |
| inv args.src.length() > 0, | |
| { | |
| c4 = args.src.peek_u32le() | |
| if 0x00 <> (LUT_CHARS[0xFF & (c4 >> 0)] | | |
| LUT_CHARS[0xFF & (c4 >> 8)] | | |
| LUT_CHARS[0xFF & (c4 >> 16)] | | |
| LUT_CHARS[0xFF & (c4 >> 24)]) { | |
| break | |
| } | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| if string_length > (0xFFFB - 4) { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length + 4) | |
| string_length = 0 | |
| continue.string_loop_outer | |
| } | |
| string_length += 4 | |
| } endwhile | |
| c = args.src.peek_u8() | |
| char = LUT_CHARS[c] | |
| if char == 0x00 { // Non-special ASCII. | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if string_length >= 0xFFFB { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: 0xFFFC) | |
| string_length = 0 | |
| continue.string_loop_outer | |
| } | |
| string_length += 1 | |
| continue.string_loop_inner | |
| } else if char == 0x01 { // '"' | |
| if string_length <> 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| } | |
| break.string_loop_outer | |
| } else if char == 0x02 { // '\\'. | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.string_loop_outer | |
| } | |
| } | |
| assert args.dst.length() > 0 | |
| if args.src.length() < 2 { | |
| if args.src.is_closed() { | |
| return "#bad backslash-escape" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| c = (args.src.peek_u16le() >> 8) as base.u8 | |
| backslash = LUT_BACKSLASHES[c] | |
| if (backslash & 0x80) <> 0 { | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| ((backslash & 0x7F) as base.u32), | |
| continued: 1, | |
| length: 2) | |
| continue.string_loop_outer | |
| } else if backslash <> 0 { | |
| if this.quirks[LUT_QUIRKY_BACKSLASHES_QUIRKS[backslash & 7]] { | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| (LUT_QUIRKY_BACKSLASHES_CHARS[backslash & 7] as base.u32), | |
| continued: 1, | |
| length: 2) | |
| continue.string_loop_outer | |
| } | |
| } else if c == 'u' { | |
| // -------- BEGIN backslash-u. | |
| if args.src.length() < 6 { | |
| if args.src.is_closed() { | |
| return "#bad backslash-escape" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| uni4_string = args.src.peek_u48le_as_u64() >> 16 | |
| uni4_value = 0 | |
| uni4_ok = 0x80 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 0)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 12 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 8)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 8 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 16)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 4 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 24)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 0 | |
| if uni4_ok == 0 { | |
| // It wasn't 4 hexadecimal digits. No-op (and | |
| // fall through to "#bad backslash-escape"). | |
| } else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) { | |
| // Not a Unicode surrogate. We're good. | |
| args.src.skip_u32_fast!(actual: 6, worst_case: 6) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| uni4_value, | |
| continued: 1, | |
| length: 6) | |
| continue.string_loop_outer | |
| } else if uni4_value >= 0xDC00 { | |
| // Low surrogate. No-op (and fall through to | |
| // "#bad backslash-escape"). | |
| } else { | |
| // High surrogate, which needs to be followed | |
| // by a "\\u1234" low surrogate. We've already | |
| // peeked 6 bytes for the high surrogate. We | |
| // need 12 in total: another 8 bytes at an | |
| // offset of 4. | |
| if args.src.length() < 12 { | |
| if args.src.is_closed() { | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.src.skip_u32_fast!(actual: 6, worst_case: 6) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 6) | |
| continue.string_loop_outer | |
| } | |
| return "#bad backslash-escape" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| uni4_string = args.src.peek_u64le_at(offset: 4) >> 16 | |
| // Look for the low surrogate's "\\u". | |
| if ((0xFF & (uni4_string >> 0)) <> '\\') or | |
| ((0xFF & (uni4_string >> 8)) <> 'u') { | |
| uni4_high_surrogate = 0 | |
| uni4_value = 0 | |
| uni4_ok = 0 | |
| } else { | |
| uni4_high_surrogate = | |
| 0x1_0000 + ((uni4_value - 0xD800) << 10) | |
| uni4_value = 0 | |
| uni4_ok = 0x80 | |
| uni4_string >>= 16 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 0)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 12 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 8)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 8 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 16)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 4 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 24)] | |
| uni4_ok &= c | |
| uni4_value |= ((c & 0x0F) as base.u32) << 0 | |
| } | |
| if (uni4_ok <> 0) and | |
| (0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) { | |
| // Emit a single token for the surrogate | |
| // pair. | |
| uni4_value -= 0xDC00 | |
| args.src.skip_u32_fast!(actual: 12, worst_case: 12) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| uni4_high_surrogate | uni4_value, | |
| continued: 1, | |
| length: 12) | |
| continue.string_loop_outer | |
| } | |
| } | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| if args.src.length() < 6 { | |
| return "#internal error: inconsistent I/O" | |
| } | |
| args.src.skip_u32_fast!(actual: 6, worst_case: 6) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 6) | |
| continue.string_loop_outer | |
| } | |
| // -------- END backslash-u. | |
| } else if (c == 'U') and | |
| this.quirks[QUIRK_ALLOW_BACKSLASH_CAPITAL_U - QUIRKS_BASE] { | |
| // -------- BEGIN backslash-capital-u. | |
| if args.src.length() < 10 { | |
| if args.src.is_closed() { | |
| return "#bad backslash-escape" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| uni8_string = args.src.peek_u64le_at(offset: 2) | |
| uni8_value = 0 | |
| uni8_ok = 0x80 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 0)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 28 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 8)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 24 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 16)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 20 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 24)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 16 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 32)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 12 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 40)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 8 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 48)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 4 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 56)] | |
| uni8_ok &= c | |
| uni8_value |= ((c & 0x0F) as base.u32) << 0 | |
| if uni8_ok == 0 { | |
| // It wasn't 8 hexadecimal digits. No-op (and | |
| // fall through to "#bad backslash-escape"). | |
| } else if (uni8_value < 0xD800) or ( | |
| (0xDFFF < uni8_value) and (uni8_value <= 0x10_FFFF)) { | |
| // Not a Unicode surrogate. We're good. | |
| args.src.skip_u32_fast!(actual: 10, worst_case: 10) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| (uni8_value & 0x1F_FFFF), | |
| continued: 1, | |
| length: 10) | |
| continue.string_loop_outer | |
| } else if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.src.skip_u32_fast!(actual: 10, worst_case: 10) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 10) | |
| continue.string_loop_outer | |
| } | |
| // -------- END backslash-capital-u. | |
| } else if (c == 'x') and | |
| this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS - QUIRKS_BASE] { | |
| // -------- BEGIN backslash-x | |
| if args.src.length() < 4 { | |
| if args.src.is_closed() { | |
| return "#bad backslash-escape" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| backslash_x_string = args.src.peek_u32le() | |
| backslash_x_ok = 0x80 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 16)] | |
| backslash_x_ok &= c | |
| backslash_x_value = ((c & 0x0F) << 4) as base.u8 | |
| c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 24)] | |
| backslash_x_ok &= c | |
| backslash_x_value = (backslash_x_value | (c & 0x0F)) as base.u8 | |
| if (backslash_x_ok == 0) or | |
| ((backslash_x_string & 0xFFFF) <> 0x785C) { | |
| // It wasn't "\\x34", for some hexadecimal | |
| // digits "34". | |
| return "#bad backslash-escape" | |
| } | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| (backslash_x_value as base.u32), | |
| continued: 1, | |
| length: 4) | |
| continue.string_loop_outer | |
| // -------- END backslash-x | |
| } | |
| return "#bad backslash-escape" | |
| } else if char == 0x03 { // 2-byte UTF-8. | |
| if args.src.length() < 2 { | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.string_loop_outer | |
| } | |
| } | |
| if args.src.is_closed() { | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| continue.string_loop_outer | |
| } | |
| return "#bad UTF-8" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| multi_byte_utf8 = args.src.peek_u16le_as_u32() | |
| if (multi_byte_utf8 & 0xC000) == 0x8000 { | |
| multi_byte_utf8 = (0x00_07C0 & (multi_byte_utf8 ~mod<< 6)) | | |
| (0x00_003F & (multi_byte_utf8 >> 8)) | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| if string_length >= 0xFFF8 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length + 2) | |
| string_length = 0 | |
| continue.string_loop_outer | |
| } | |
| string_length += 2 | |
| continue.string_loop_inner | |
| } | |
| } else if char == 0x04 { // 3-byte UTF-8. | |
| if args.src.length() < 3 { | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.string_loop_outer | |
| } | |
| } | |
| if args.src.is_closed() { | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| continue.string_loop_outer | |
| } | |
| return "#bad UTF-8" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| multi_byte_utf8 = args.src.peek_u24le_as_u32() | |
| if (multi_byte_utf8 & 0xC0_C000) == 0x80_8000 { | |
| multi_byte_utf8 = (0x00_F000 & (multi_byte_utf8 ~mod<< 12)) | | |
| (0x00_0FC0 & (multi_byte_utf8 >> 2)) | | |
| (0x00_003F & (multi_byte_utf8 >> 16)) | |
| if (0x07FF < multi_byte_utf8) and | |
| ((multi_byte_utf8 < 0xD800) or (0xDFFF < multi_byte_utf8)) { | |
| args.src.skip_u32_fast!(actual: 3, worst_case: 3) | |
| if string_length >= 0xFFF8 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length + 3) | |
| string_length = 0 | |
| continue.string_loop_outer | |
| } | |
| string_length += 3 | |
| continue.string_loop_inner | |
| } | |
| } | |
| } else if char == 0x05 { // 4-byte UTF-8. | |
| if args.src.length() < 4 { | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.string_loop_outer | |
| } | |
| } | |
| if args.src.is_closed() { | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| continue.string_loop_outer | |
| } | |
| return "#bad UTF-8" | |
| } | |
| yield? base."$short read" | |
| continue.string_loop_outer | |
| } | |
| multi_byte_utf8 = args.src.peek_u32le() | |
| if (multi_byte_utf8 & 0xC0C0_C000) == 0x8080_8000 { | |
| multi_byte_utf8 = (0x1C_0000 & (multi_byte_utf8 ~mod<< 18)) | | |
| (0x03_F000 & (multi_byte_utf8 ~mod<< 4)) | | |
| (0x00_0FC0 & (multi_byte_utf8 >> 10)) | | |
| (0x00_003F & (multi_byte_utf8 >> 24)) | |
| if (0xFFFF < multi_byte_utf8) and (multi_byte_utf8 <= 0x10_FFFF) { | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| if string_length >= 0xFFF8 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length + 4) | |
| string_length = 0 | |
| continue.string_loop_outer | |
| } | |
| string_length += 4 | |
| continue.string_loop_inner | |
| } | |
| } | |
| } | |
| if string_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, | |
| continued: 1, | |
| length: string_length) | |
| string_length = 0 | |
| if args.dst.length() <= 0 { | |
| continue.string_loop_outer | |
| } | |
| } | |
| if (char & 0x80) <> 0 { | |
| if this.quirks[QUIRK_ALLOW_ASCII_CONTROL_CODES - QUIRKS_BASE] { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| ((char & 0x7F) as base.u32), | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| continue.string_loop_outer | |
| } | |
| if char == 0x8A { | |
| return "#bad new-line in a string" | |
| } | |
| return "#bad C0 control code" | |
| } | |
| if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | | |
| base.UNICODE__REPLACEMENT_CHARACTER, | |
| continued: 1, | |
| length: 1) | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| continue.string_loop_outer | |
| } | |
| return "#bad UTF-8" | |
| } endwhile.string_loop_inner | |
| } endwhile.string_loop_outer | |
| // Emit the trailing '"'. | |
| while true { | |
| if args.src.length() <= 0 { | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| continue | |
| } | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue | |
| } | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRING << 21) | | |
| base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | | |
| base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | | |
| base.TOKEN__VBD__STRING__DEFINITELY_ASCII | | |
| base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, | |
| continued: 0, | |
| length: 1) | |
| break | |
| } endwhile | |
| // As above, expect must have contained EXPECT_STRING. If it didn't | |
| // also contain EXPECT_NUMBER (excluding EXPECT_COMMENT) then we | |
| // were parsing an object key and the next token should be ':'. | |
| if 0 == (expect & ((1 as base.u32) << CLASS_NUMBER)) { | |
| expect = EXPECT_COLON | |
| continue.outer | |
| } | |
| break.goto_parsed_a_leaf_value | |
| // -------- END parse strings. | |
| } else if class == CLASS_COMMA { | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| // The ',' is punctuation (filler). | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__PUNCTUATION, | |
| continued: 0, | |
| length: 1) | |
| // What's valid after a comma depends on whether or not we're in an | |
| // array or an object. | |
| if 0 == (expect & ((1 as base.u32) << CLASS_CLOSE_SQUARE_BRACKET)) { | |
| if this.quirks[QUIRK_ALLOW_EXTRA_COMMA - QUIRKS_BASE] { | |
| expect = EXPECT_STRING | EXPECT_CLOSE_CURLY_BRACE | |
| } else { | |
| expect = EXPECT_STRING | |
| } | |
| } else { | |
| if this.quirks[QUIRK_ALLOW_EXTRA_COMMA - QUIRKS_BASE] { | |
| expect = EXPECT_VALUE | EXPECT_CLOSE_SQUARE_BRACKET | |
| } else { | |
| expect = EXPECT_VALUE | |
| } | |
| } | |
| continue.outer | |
| } else if class == CLASS_COLON { | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| // The ':' is punctuation (filler). | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__PUNCTUATION, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_VALUE | |
| continue.outer | |
| } else if class == CLASS_NUMBER { | |
| // -------- BEGIN parse numbers. | |
| while true, | |
| pre args.dst.length() > 0, | |
| { | |
| number_length = this.decode_number!(src: args.src) | |
| number_status = number_length >> 8 | |
| vminor = (base.TOKEN__VBC__NUMBER << 21) | | |
| base.TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT | | |
| base.TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED | | |
| base.TOKEN__VBD__NUMBER__FORMAT_TEXT | |
| if (number_length & 0x80) <> 0 { | |
| vminor = (base.TOKEN__VBC__NUMBER << 21) | | |
| base.TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT | | |
| base.TOKEN__VBD__NUMBER__FORMAT_TEXT | |
| } | |
| number_length = number_length & 0x7F | |
| if number_status == 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: vminor, | |
| continued: 0, | |
| length: number_length) | |
| break | |
| } | |
| while number_length > 0 { | |
| number_length -= 1 | |
| if args.src.can_undo_byte() { | |
| args.src.undo_byte!() | |
| } else { | |
| return "#internal error: inconsistent I/O" | |
| } | |
| } endwhile | |
| if number_status == 1 { | |
| if this.quirks[QUIRK_ALLOW_INF_NAN_NUMBERS - QUIRKS_BASE] { | |
| this.decode_inf_nan?(dst: args.dst, src: args.src) | |
| break | |
| } | |
| return "#bad input" | |
| } else if number_status == 2 { | |
| return "#unsupported number length" | |
| } else { | |
| yield? base."$short read" | |
| while args.dst.length() <= 0, | |
| post args.dst.length() > 0, | |
| { | |
| yield? base."$short write" | |
| } endwhile | |
| } | |
| } endwhile | |
| break.goto_parsed_a_leaf_value | |
| // -------- END parse numbers. | |
| } else if class == CLASS_OPEN_CURLY_BRACE { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_NONE | | |
| base.TOKEN__VBD__STRUCTURE__TO_DICT | |
| if depth == 0 { | |
| // No-op. | |
| } else if 0 <> (expect_after_value & ((1 as base.u32) << CLASS_CLOSE_CURLY_BRACE)) { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_DICT | | |
| base.TOKEN__VBD__STRUCTURE__TO_DICT | |
| } else { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_LIST | | |
| base.TOKEN__VBD__STRUCTURE__TO_DICT | |
| } | |
| if depth >= 1024 { | |
| return "#unsupported recursion depth" | |
| } | |
| stack_byte = depth / 32 | |
| stack_bit = depth & 31 | |
| this.stack[stack_byte] |= (1 as base.u32) << stack_bit | |
| depth += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: vminor, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING | |
| expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA | |
| continue.outer | |
| } else if class == CLASS_CLOSE_CURLY_BRACE { | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if depth <= 1 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_DICT | | |
| base.TOKEN__VBD__STRUCTURE__TO_NONE, | |
| continued: 0, | |
| length: 1) | |
| break.outer | |
| } | |
| depth -= 1 | |
| stack_byte = (depth - 1) / 32 | |
| stack_bit = (depth - 1) & 31 | |
| if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_DICT | | |
| base.TOKEN__VBD__STRUCTURE__TO_LIST, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA | |
| expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA | |
| } else { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_DICT | | |
| base.TOKEN__VBD__STRUCTURE__TO_DICT, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA | |
| expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA | |
| } | |
| continue.outer | |
| } else if class == CLASS_OPEN_SQUARE_BRACKET { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_NONE | | |
| base.TOKEN__VBD__STRUCTURE__TO_LIST | |
| if depth == 0 { | |
| // No-op. | |
| } else if 0 <> (expect_after_value & ((1 as base.u32) << CLASS_CLOSE_CURLY_BRACE)) { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_DICT | | |
| base.TOKEN__VBD__STRUCTURE__TO_LIST | |
| } else { | |
| vminor = (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__PUSH | | |
| base.TOKEN__VBD__STRUCTURE__FROM_LIST | | |
| base.TOKEN__VBD__STRUCTURE__TO_LIST | |
| } | |
| if depth >= 1024 { | |
| return "#unsupported recursion depth" | |
| } | |
| stack_byte = depth / 32 | |
| stack_bit = depth & 31 | |
| this.stack[stack_byte] &= 0xFFFF_FFFF ^ ((1 as base.u32) << stack_bit) | |
| depth += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: vminor, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE | |
| expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA | |
| continue.outer | |
| } else if class == CLASS_CLOSE_SQUARE_BRACKET { | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if depth <= 1 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_LIST | | |
| base.TOKEN__VBD__STRUCTURE__TO_NONE, | |
| continued: 0, | |
| length: 1) | |
| break.outer | |
| } | |
| depth -= 1 | |
| stack_byte = (depth - 1) / 32 | |
| stack_bit = (depth - 1) & 31 | |
| if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_LIST | | |
| base.TOKEN__VBD__STRUCTURE__TO_LIST, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA | |
| expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA | |
| } else { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__STRUCTURE << 21) | | |
| base.TOKEN__VBD__STRUCTURE__POP | | |
| base.TOKEN__VBD__STRUCTURE__FROM_LIST | | |
| base.TOKEN__VBD__STRUCTURE__TO_DICT, | |
| continued: 0, | |
| length: 1) | |
| expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA | |
| expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA | |
| } | |
| continue.outer | |
| } else if class == CLASS_FALSE { | |
| match = args.src.match7(a: '\x05false'le) | |
| if match == 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__LITERAL << 21) | | |
| base.TOKEN__VBD__LITERAL__FALSE, | |
| continued: 0, | |
| length: 5) | |
| if args.src.length() < 5 { | |
| return "#internal error: inconsistent I/O" | |
| } | |
| args.src.skip_u32_fast!(actual: 5, worst_case: 5) | |
| break.goto_parsed_a_leaf_value | |
| } else if match == 1 { | |
| yield? base."$short read" | |
| continue.outer | |
| } | |
| } else if class == CLASS_TRUE { | |
| match = args.src.match7(a: '\x04true'le) | |
| if match == 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__LITERAL << 21) | | |
| base.TOKEN__VBD__LITERAL__TRUE, | |
| continued: 0, | |
| length: 4) | |
| if args.src.length() < 4 { | |
| return "#internal error: inconsistent I/O" | |
| } | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| break.goto_parsed_a_leaf_value | |
| } else if match == 1 { | |
| yield? base."$short read" | |
| continue.outer | |
| } | |
| } else if class == CLASS_NULL_NAN_INF { | |
| match = args.src.match7(a: '\x04null'le) | |
| if match == 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__LITERAL << 21) | | |
| base.TOKEN__VBD__LITERAL__NULL, | |
| continued: 0, | |
| length: 4) | |
| if args.src.length() < 4 { | |
| return "#internal error: inconsistent I/O" | |
| } | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| break.goto_parsed_a_leaf_value | |
| } else if match == 1 { | |
| yield? base."$short read" | |
| continue.outer | |
| } | |
| if this.quirks[QUIRK_ALLOW_INF_NAN_NUMBERS - QUIRKS_BASE] { | |
| this.decode_inf_nan?(dst: args.dst, src: args.src) | |
| break.goto_parsed_a_leaf_value | |
| } | |
| } else if class == CLASS_COMMENT { | |
| if this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] or | |
| this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] { | |
| this.decode_comment?(dst: args.dst, src: args.src) | |
| if this.comment_type > 0 { | |
| continue.outer | |
| } | |
| } | |
| } | |
| return "#bad input" | |
| }} endwhile.goto_parsed_a_leaf_value | |
| // We've just parsed a leaf (non-container) value: literal (null, | |
| // false, true), number or string. | |
| if depth == 0 { | |
| break.outer | |
| } | |
| expect = expect_after_value | |
| } endwhile.outer | |
| if this.quirks[QUIRK_ALLOW_TRAILING_FILLER - QUIRKS_BASE] or | |
| this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] { | |
| this.decode_trailer?(dst: args.dst, src: args.src) | |
| } | |
| this.end_of_data = true | |
| } | |
| pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { | |
| var c : base.u8 | |
| var n : base.u32[..= 0x3FF] | |
| var floating_point : base.u32[..= 0x80] | |
| while.goto_done true {{ | |
| n = 0 | |
| // Peek. | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| break.goto_done | |
| } | |
| c = args.src.peek_u8() | |
| // Scan the optional minus sign. | |
| if c <> '-' { | |
| assert args.src.length() > 0 | |
| assert n <= 1 | |
| } else { | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| // Peek. | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| n |= 0x100 // A '-' without digits is invalid. | |
| break.goto_done | |
| } | |
| c = args.src.peek_u8() | |
| assert args.src.length() > 0 | |
| assert n <= 1 | |
| } | |
| // Scan the opening digits. | |
| if c == '0' { | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| assert n <= 99 | |
| } else { | |
| n = this.decode_digits!(src: args.src, n: n) | |
| if n > 99 { | |
| break.goto_done | |
| } | |
| assert n <= 99 | |
| } | |
| // Peek. | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| break.goto_done | |
| } | |
| c = args.src.peek_u8() | |
| // Scan the optional fraction. | |
| if c <> '.' { | |
| assert args.src.length() > 0 | |
| assert n <= 99 | |
| } else { | |
| if n >= 99 { | |
| n |= 0x200 | |
| break.goto_done | |
| } | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| floating_point = 0x80 | |
| n = this.decode_digits!(src: args.src, n: n) | |
| if n > 99 { | |
| break.goto_done | |
| } | |
| // Peek. | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| break.goto_done | |
| } | |
| c = args.src.peek_u8() | |
| assert args.src.length() > 0 | |
| assert n <= 99 | |
| } | |
| // Scan the optional 'E' or 'e'. | |
| if (c <> 'E') and (c <> 'e') { | |
| break.goto_done | |
| } | |
| if n >= 99 { | |
| n |= 0x200 | |
| break.goto_done | |
| } | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| floating_point = 0x80 | |
| assert n <= 99 | |
| // Peek. | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| n |= 0x100 // An 'E' or 'e' without digits is invalid. | |
| break.goto_done | |
| } | |
| c = args.src.peek_u8() | |
| // Scan the optional '+' or '-'. | |
| if (c <> '+') and (c <> '-') { | |
| assert n <= 99 | |
| } else { | |
| if n >= 99 { | |
| n |= 0x200 | |
| break.goto_done | |
| } | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| assert n <= 99 | |
| } | |
| // Scan the exponent digits. | |
| n = this.decode_digits!(src: args.src, n: n) | |
| break.goto_done | |
| }} endwhile.goto_done | |
| return n | floating_point | |
| } | |
| pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 99]) base.u32[..= 0x3FF] { | |
| var c : base.u8 | |
| var n : base.u32[..= 0x3FF] | |
| n = args.n | |
| while true { | |
| if args.src.length() <= 0 { | |
| if not args.src.is_closed() { | |
| n |= 0x300 | |
| } | |
| break | |
| } | |
| c = args.src.peek_u8() | |
| if 0x00 == LUT_DECIMAL_DIGITS[c] { | |
| break | |
| } | |
| // Cap DECODER_NUMBER_LENGTH_MAX_INCL at an arbitrary value, 99. The | |
| // caller's src.data.len should therefore be at least 100, also known | |
| // as DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL. | |
| // | |
| // An example of a JSON number that is 81 bytes long is: | |
| // https://github.com/nst/JSONTestSuite/blob/master/test_parsing/y_number_double_close_to_zero.json | |
| // | |
| // Note that 99 (in hex, 0x63) is less than 0x80, so we can use 0x80 as | |
| // a flag bit in func decoder.decode_number. | |
| if n >= 99 { | |
| n |= 0x200 | |
| break | |
| } | |
| n += 1 | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| } endwhile | |
| if n == args.n { | |
| n |= 0x100 | |
| } | |
| return n | |
| } | |
| pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { | |
| var c : base.u8 | |
| var u : base.u32 | |
| this.allow_leading_ars = | |
| this.quirks[QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR - QUIRKS_BASE] | |
| this.allow_leading_ubom = | |
| this.quirks[QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK - QUIRKS_BASE] | |
| while this.allow_leading_ars or this.allow_leading_ubom { | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue | |
| } | |
| if args.src.length() <= 0 { | |
| if args.src.is_closed() { | |
| break | |
| } | |
| yield? base."$short read" | |
| continue | |
| } | |
| c = args.src.peek_u8() | |
| if (c == 0x1E) and this.allow_leading_ars { | |
| this.allow_leading_ars = false | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, value_minor: 0, continued: 0, length: 1) | |
| continue | |
| } else if (c == 0xEF) and this.allow_leading_ubom { | |
| if args.src.length() < 3 { | |
| if args.src.is_closed() { | |
| break | |
| } | |
| yield? base."$short read" | |
| continue | |
| } | |
| u = args.src.peek_u24le_as_u32() | |
| if u == 0xBF_BBEF { | |
| this.allow_leading_ubom = false | |
| args.src.skip_u32_fast!(actual: 3, worst_case: 3) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, value_minor: 0, continued: 0, length: 3) | |
| continue | |
| } | |
| } | |
| break | |
| } endwhile | |
| } | |
| pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { | |
| var c : base.u8 | |
| var c2 : base.u16 | |
| var length : base.u32[..= 0xFFFD] | |
| this.comment_type = 0 | |
| while (args.dst.length() <= 0) or (args.src.length() <= 1), | |
| post args.dst.length() > 0, | |
| post args.src.length() > 1, | |
| { | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue | |
| } | |
| if args.src.is_closed() { | |
| return ok | |
| } | |
| yield? base."$short read" | |
| } endwhile | |
| c2 = args.src.peek_u16le() | |
| if (c2 == '/*'le) and this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] { | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| length = 2 | |
| while.comment_block true, | |
| pre args.dst.length() > 0, | |
| { | |
| if args.src.length() <= 1 { | |
| if length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_BLOCK, | |
| continued: 1, | |
| length: length) | |
| } | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| while args.dst.length() <= 0, | |
| post args.dst.length() > 0, | |
| { | |
| yield? base."$short write" | |
| } endwhile | |
| length = 0 | |
| continue.comment_block | |
| } | |
| c2 = args.src.peek_u16le() | |
| if c2 == '*/'le { | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_BLOCK, | |
| continued: 0, | |
| length: length + 2) | |
| this.comment_type = 1 | |
| return ok | |
| } | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if length >= 0xFFFD { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_BLOCK, | |
| continued: 1, | |
| length: length + 1) | |
| while args.dst.length() <= 0, | |
| post args.dst.length() > 0, | |
| { | |
| yield? base."$short write" | |
| } endwhile | |
| length = 0 | |
| continue.comment_block | |
| } | |
| length += 1 | |
| } endwhile.comment_block | |
| } else if (c2 == '//'le) and this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] { | |
| args.src.skip_u32_fast!(actual: 2, worst_case: 2) | |
| length = 2 | |
| while.comment_line true, | |
| pre args.dst.length() > 0, | |
| { | |
| if args.src.length() <= 0 { | |
| if args.src.is_closed() { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_LINE, | |
| continued: 0, | |
| length: length) | |
| this.comment_type = 2 | |
| return ok | |
| } else if length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_LINE, | |
| continued: 1, | |
| length: length) | |
| } | |
| yield? base."$short read" | |
| while args.dst.length() <= 0, | |
| post args.dst.length() > 0, | |
| { | |
| yield? base."$short write" | |
| } endwhile | |
| length = 0 | |
| continue.comment_line | |
| } | |
| c = args.src.peek_u8() | |
| if c == '\n' { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_LINE, | |
| continued: 0, | |
| length: length) | |
| this.comment_type = 2 | |
| return ok | |
| } | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if length >= 0xFFFD { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__FILLER << 21) | | |
| base.TOKEN__VBD__FILLER__COMMENT_LINE, | |
| continued: 1, | |
| length: length + 1) | |
| while args.dst.length() <= 0, | |
| post args.dst.length() > 0, | |
| { | |
| yield? base."$short write" | |
| } endwhile | |
| length = 0 | |
| continue.comment_line | |
| } | |
| length += 1 | |
| } endwhile.comment_line | |
| } | |
| } | |
| pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { | |
| var c4 : base.u32 | |
| var neg : base.u32[..= 1] | |
| while true { | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue | |
| } | |
| if args.src.length() <= 2 { | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| continue | |
| } | |
| // Bitwise or'ing with 0x20 converts upper case ASCII to lower case. | |
| c4 = args.src.peek_u24le_as_u32() | |
| if (c4 | 0x20_2020) == 'inf'le { | |
| if args.src.length() > 7 { | |
| if (args.src.peek_u64le() | 0x2020_2020_2020_2020) == 'infinity'le { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| base.TOKEN__VBD__NUMBER__CONTENT_POS_INF, | |
| continued: 0, | |
| length: 8) | |
| args.src.skip_u32_fast!(actual: 8, worst_case: 8) | |
| return ok | |
| } | |
| } else if not args.src.is_closed() { | |
| yield? base."$short read" | |
| continue | |
| } | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| base.TOKEN__VBD__NUMBER__CONTENT_POS_INF, | |
| continued: 0, | |
| length: 3) | |
| args.src.skip_u32_fast!(actual: 3, worst_case: 3) | |
| return ok | |
| } else if (c4 | 0x20_2020) == 'nan'le { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| base.TOKEN__VBD__NUMBER__CONTENT_POS_NAN, | |
| continued: 0, | |
| length: 3) | |
| args.src.skip_u32_fast!(actual: 3, worst_case: 3) | |
| return ok | |
| } else if (c4 & 0xFF) == '+' { | |
| neg = 0 | |
| } else if (c4 & 0xFF) == '-' { | |
| neg = 1 | |
| } else { | |
| return "#bad input" | |
| } | |
| if args.src.length() <= 3 { | |
| if args.src.is_closed() { | |
| return "#bad input" | |
| } | |
| yield? base."$short read" | |
| continue | |
| } | |
| c4 = args.src.peek_u32le() >> 8 | |
| if (c4 | 0x20_2020) == 'inf'le { | |
| if args.src.length() > 8 { | |
| if (args.src.peek_u64le_at(offset: 1) | 0x2020_2020_2020_2020) == 'infinity'le { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| (base.TOKEN__VBD__NUMBER__CONTENT_POS_INF >> neg), | |
| continued: 0, | |
| length: 9) | |
| args.src.skip_u32_fast!(actual: 9, worst_case: 9) | |
| return ok | |
| } | |
| } else if not args.src.is_closed() { | |
| yield? base."$short read" | |
| continue | |
| } | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| (base.TOKEN__VBD__NUMBER__CONTENT_POS_INF >> neg), | |
| continued: 0, | |
| length: 4) | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| return ok | |
| } else if (c4 | 0x20_2020) == 'nan'le { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, | |
| value_minor: (base.TOKEN__VBC__NUMBER << 21) | | |
| (base.TOKEN__VBD__NUMBER__CONTENT_POS_NAN >> neg), | |
| continued: 0, | |
| length: 4) | |
| args.src.skip_u32_fast!(actual: 4, worst_case: 4) | |
| return ok | |
| } | |
| return "#bad input" | |
| } endwhile | |
| } | |
| pri func decoder.decode_trailer?(dst: base.token_writer, src: base.io_reader) { | |
| var c : base.u8 | |
| var whitespace_length : base.u32[..= 0xFFFE] | |
| if this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] { | |
| this.trailer_stop = '\n' | |
| } else { | |
| this.trailer_stop = 0 | |
| } | |
| while.outer true { | |
| if args.dst.length() <= 0 { | |
| yield? base."$short write" | |
| continue.outer | |
| } | |
| whitespace_length = 0 | |
| while.inner true, | |
| pre args.dst.length() > 0, | |
| { | |
| if args.src.length() <= 0 { | |
| if whitespace_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, value_minor: 0, continued: 0, length: whitespace_length) | |
| } | |
| if args.src.is_closed() { | |
| break.outer | |
| } | |
| yield? base."$short read" | |
| continue.outer | |
| } | |
| c = args.src.peek_u8() | |
| if LUT_CLASSES[c] <> CLASS_WHITESPACE { | |
| if whitespace_length > 0 { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, value_minor: 0, continued: 0, length: whitespace_length) | |
| } | |
| if this.trailer_stop > 0 { | |
| return "#bad input" | |
| } | |
| this.decode_comment?(dst: args.dst, src: args.src) | |
| if this.comment_type > 0 { | |
| continue.outer | |
| } | |
| return ok | |
| } | |
| args.src.skip_u32_fast!(actual: 1, worst_case: 1) | |
| if (whitespace_length >= 0xFFFE) or (c == this.trailer_stop) { | |
| args.dst.write_simple_token_fast!( | |
| value_major: 0, value_minor: 0, continued: 0, length: whitespace_length + 1) | |
| if c == this.trailer_stop { | |
| return ok | |
| } | |
| continue.outer | |
| } | |
| whitespace_length += 1 | |
| } endwhile.inner | |
| } endwhile.outer | |
| } |