Tweak wuffs_base__token bitmask assignments again
diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 76f8b62..389cab6 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h
@@ -53,6 +53,8 @@ return ret; } + // -------- + #define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF @@ -67,54 +69,82 @@ #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 + // -------- + #define WUFFS_BASE__TOKEN__VBC__FILLER 0 #define WUFFS_BASE__TOKEN__VBC__STRING 1 -#define WUFFS_BASE__TOKEN__VBC__BYTES 2 -#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3 -#define WUFFS_BASE__TOKEN__VBC__NUMBER 4 -#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 3 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4 +// -------- + +// INCOMPLETE means that this token combines with the following token. +// +// For example, tokenizing a comment that is longer than the source buffer can +// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE. +// +// By convention, whitespace is not marked incomplete. Two whitespace tokens of +// length 30 are equivalent to one whitespace token of length 60. +// // Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. +// VBD_FILLER and VBD_STRING. #define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001 + #define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040 +// -------- + +// INCOMPLETE means that this token combines with the following token. +// +// For example, tokenizing a string that is longer than the source buffer can +// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE. +// // Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. +// VBD_FILLER and VBD_STRING. #define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010 -// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. -#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001 -// "D_DST_S_SRC" means that it takes S source bytes (possibly padded) to -// produce D destination bytes. For example, -// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X means that the source -// looks like "\\x23\\x67\\xAB", where 12 src bytes encode 3 dst bytes. -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080 -#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100 -#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200 -#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400 +// "DEFINITELY_FOO" means that the source bytes (and also the destination +// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means +// that the lack of the bit is conservative: it is valid for all-ASCII strings +// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set. +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 +// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly +// padded) produces multiples of D destination bytes. For example, +// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where +// 12 src bytes encode 3 dst bytes. +// +// When src is the empty string, multiple conversion algorithms are applicable +// (so these bits are not necessarily mutually exclusive), all producing the +// same empty dst string. +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000 + + // -------- + + // UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit. + // All 21 bits are used to hold the Unicode code point. Such tokens preserve + // the INCOMPLETEness of the previous token (if a FILLER or STRING). + +#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF + + // -------- #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801 + // For a source string of "123" or "0x9A", it is valid for a tokenizer to // return any one of: // - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT. @@ -128,7 +158,18 @@ #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004 #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008 -#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF + // -------- + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + +// -------- static inline uint64_t // wuffs_base__token__value(const wuffs_base__token* t) {
diff --git a/internal/cgen/data.go b/internal/cgen/data.go index e3d60de..90532ea 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go
@@ -288,12 +288,25 @@ const baseTokenPublicH = "" + "// ---------------- Tokens\n\ntypedef struct {\n // The repr is divided as:\n // - Bits 63 .. 40 (24 bits) is the major value.\n // - Bits 39 .. 16 (24 bits) is the minor value.\n // - Bits 15 .. 0 (16 bits) is the length.\n //\n // The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If\n // zero (special cased for Wuffs' built-in \"base\" package) then the minor\n // value is further sub-divided:\n // - Bits 39 .. 37 ( 3 bits) is the value_base_category.\n // - Bits 36 .. 16 (21 bits) is the value_base_detail.\n //\n // In particular, at 21 bits, the value_base_detail can hold every valid\n // Unicode code point.\n //\n // If the major value is non-zero then the minor value has whatever arbitrary\n // meaning the tokenizer's package assigns to it.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inlin" + - "e uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__BYTES 2\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3\n#define WUFFS_BASE" + - "__TOKEN__VBC__NUMBER 4\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001\n// \"D_DST_S_SRC\" means that it takes S source bytes (possibly padded) to\n// produce D destination bytes. For example,\n// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH" + - "_X means that the source\n// looks like \"\\\\x23\\\\x67\\\\xAB\", where 12 src bytes encode 3 dst bytes.\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080\n#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRU" + - "CTURE__TO_DICT 0x04000\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008\n\n#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF\n\nstatic inline uint64_t " + - " //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t " + - " //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + + "e uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " + + "" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n " + + "" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 3\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4\n\n" + + "" + + "// --------\n\n// INCOMPLETE means that this token combines with the following token.\n//\n// For example, tokenizing a comment that is longer than the source buffer can\n// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE.\n//\n// By convention, whitespace is not marked incomplete. Two whitespace tokens of\n// length 30 are equivalent to one whitespace token of length 60.\n//\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER and VBD_STRING.\n#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040\n\n" + + "" + + "// --------\n\n// INCOMPLETE means that this token combines with the following token.\n//\n// For example, tokenizing a string that is longer than the source buffer can\n// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE.\n//\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER and VBD_STRING.\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n\n// \"DEFINITELY_FOO\" means that the source bytes (and also the destination\n// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means\n// that the lack of the bit is conservative: it is valid for all-ASCII strings\n// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a sourc" + + "e like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n " + + "" + + "// --------\n\n // UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit.\n // All 21 bits are used to hold the Unicode code point. Such tokens preserve\n // the INCOMPLETEness of the previous token (if a FILLER or STRING).\n\n#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF\n\n " + + "" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008\n\n " + + "" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + + "" + + "// --------\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETA" + + "IL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + "" + "// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" + "" +
diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f2ab00b..aa19fa3 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c
@@ -1515,6 +1515,8 @@ return ret; } + // -------- + #define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF @@ -1529,54 +1531,82 @@ #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 + // -------- + #define WUFFS_BASE__TOKEN__VBC__FILLER 0 #define WUFFS_BASE__TOKEN__VBC__STRING 1 -#define WUFFS_BASE__TOKEN__VBC__BYTES 2 -#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3 -#define WUFFS_BASE__TOKEN__VBC__NUMBER 4 -#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 3 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4 +// -------- + +// INCOMPLETE means that this token combines with the following token. +// +// For example, tokenizing a comment that is longer than the source buffer can +// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE. +// +// By convention, whitespace is not marked incomplete. Two whitespace tokens of +// length 30 are equivalent to one whitespace token of length 60. +// // Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. +// VBD_FILLER and VBD_STRING. #define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001 + #define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040 +// -------- + +// INCOMPLETE means that this token combines with the following token. +// +// For example, tokenizing a string that is longer than the source buffer can +// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE. +// // Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. +// VBD_FILLER and VBD_STRING. #define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010 -// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between -// VBD_FILLER, VBD_STRING and VBD_BYTES. -#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001 -// "D_DST_S_SRC" means that it takes S source bytes (possibly padded) to -// produce D destination bytes. For example, -// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X means that the source -// looks like "\\x23\\x67\\xAB", where 12 src bytes encode 3 dst bytes. -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040 -#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080 -#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100 -#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200 -#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400 +// "DEFINITELY_FOO" means that the source bytes (and also the destination +// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means +// that the lack of the bit is conservative: it is valid for all-ASCII strings +// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set. +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 +// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly +// padded) produces multiples of D destination bytes. For example, +// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where +// 12 src bytes encode 3 dst bytes. +// +// When src is the empty string, multiple conversion algorithms are applicable +// (so these bits are not necessarily mutually exclusive), all producing the +// same empty dst string. +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000 + + // -------- + + // UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit. + // All 21 bits are used to hold the Unicode code point. Such tokens preserve + // the INCOMPLETEness of the previous token (if a FILLER or STRING). + +#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF + + // -------- #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401 #define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801 + // For a source string of "123" or "0x9A", it is valid for a tokenizer to // return any one of: // - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT. @@ -1590,7 +1620,18 @@ #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004 #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008 -#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF + // -------- + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + +// -------- static inline uint64_t // wuffs_base__token__value(const wuffs_base__token* t) { @@ -18996,7 +19037,7 @@ goto label__0__continue; } *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097153)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(2129713)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__0__break; } @@ -19014,7 +19055,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097153)) + (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_string_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19035,7 +19076,7 @@ if (v_c <= 34) { if (v_c == 34) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097152)) + (((uint64_t)(2097696)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_string_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19044,7 +19085,7 @@ } else if (v_c < 32) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097153)) + (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_string_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19057,7 +19098,7 @@ } else if (v_c == 92) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097153)) + (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_string_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19084,7 +19125,7 @@ if (v_c > 0) { (iop_a_src += 2, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((10485760 | ((uint64_t)((v_c & 127)))))) + (((uint64_t)((4194304 | ((uint64_t)((v_c & 127)))))) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop__continue; @@ -19125,7 +19166,7 @@ } else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) { (iop_a_src += 6, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((10485760 | ((uint64_t)(v_uni4_value))))) + (((uint64_t)((4194304 | ((uint64_t)(v_uni4_value))))) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop__continue; @@ -19180,8 +19221,8 @@ (iop_a_src += 8, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)( - (10485760 | ((uint64_t)((v_uni4_high_surrogate + - v_uni4_value)))))) + (4194304 | ((uint64_t)((v_uni4_high_surrogate + + v_uni4_value)))))) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(12)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19207,7 +19248,7 @@ (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_string_length >= 65534) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097153)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; goto label__string_loop__continue; @@ -19274,9 +19315,9 @@ iop_a_src = a_src->data.ptr + a_src->meta.ri; } v_number_status = (v_number_length >> 8); - v_token_value = 8388612; + v_token_value = 6291460; if ((v_number_length & 128) != 0) { - v_token_value = 8388610; + v_token_value = 6291458; } v_number_length = (v_number_length & 127); if (v_number_status == 0) { @@ -19318,12 +19359,12 @@ label__2__break:; goto label__goto_parsed_a_leaf_value__break; } else if (v_class == 5) { - v_token_value = 6307857; + v_token_value = 8405009; if (v_depth == 0) { } else if (0 != (v_expect_after_value & 64)) { - v_token_value = 6307905; + v_token_value = 8405057; } else { - v_token_value = 6307873; + v_token_value = 8405025; } if (v_depth >= 1024) { status = wuffs_base__make_status( @@ -19346,7 +19387,7 @@ (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_depth <= 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6295618)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8392770)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__outer__break; } @@ -19356,25 +19397,25 @@ if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6299714)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8396866)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 260; v_expect_after_value = 260; } else { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6307906)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8405058)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 68; v_expect_after_value = 68; } goto label__outer__continue; } else if (v_class == 7) { - v_token_value = 6299665; + v_token_value = 8396817; if (v_depth == 0) { } else if (0 != (v_expect_after_value & 64)) { - v_token_value = 6299713; + v_token_value = 8396865; } else { - v_token_value = 6299681; + v_token_value = 8396833; } if (v_depth >= 1024) { status = wuffs_base__make_status( @@ -19397,7 +19438,7 @@ (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_depth <= 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6295586)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8392738)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__outer__break; } @@ -19407,13 +19448,13 @@ if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6299682)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8396834)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 260; v_expect_after_value = 260; } else { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6307874)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(8405026)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 68; v_expect_after_value = 68; @@ -19424,7 +19465,7 @@ 111546413966853); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8389633)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(6292481)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 5) { status = wuffs_base__make_status( @@ -19444,7 +19485,7 @@ 435762131972); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8390657)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(6293505)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { status = wuffs_base__make_status( @@ -19464,7 +19505,7 @@ 465676103172); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8389121)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(6291969)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { status = wuffs_base__make_status(
diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 20ee6a8..b7f6387 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs
@@ -153,9 +153,10 @@ length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) - // If the string contents starts with an escape code, issue a - // zero-length string token (one with value_base_category 3) - // before issuing a token with value_base_category 4. + // If the string contents starts with an escape code, issue an + // incomplete zero-length string token (with + // WUFFS_BASE__TOKEN__VBC__STRING) before issuing a token with + // WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT. while true { if args.src.available() <= 0 { if args.src.is_closed() { @@ -172,7 +173,7 @@ continue } args.dst.write_fast_token!( - value: 0x20_0001, + value: 0x20_7F31, length: 0) break } @@ -190,7 +191,7 @@ if args.src.available() <= 0 { if string_length > 0 { args.dst.write_fast_token!( - value: 0x20_0001, + value: 0x20_0221, length: string_length as base.u64) string_length = 0 } @@ -206,7 +207,7 @@ if c <= 0x22 { // 0x22 is '"'. if c == 0x22 { // 0x22 is '"'. args.dst.write_fast_token!( - value: 0x20_0000, + value: 0x20_0220, length: string_length as base.u64) string_length = 0 break.string_loop @@ -214,7 +215,7 @@ } else if c < 0x20 { // 0x20 is ' '. if string_length > 0 { args.dst.write_fast_token!( - value: 0x20_0001, + value: 0x20_0221, length: string_length as base.u64) string_length = 0 } @@ -224,7 +225,7 @@ } else if c == 0x5C { // 0x5C is '\\'. if string_length > 0 { args.dst.write_fast_token!( - value: 0x20_0001, + value: 0x20_0221, length: string_length as base.u64) string_length = 0 if args.dst.available() <= 0 { @@ -246,7 +247,7 @@ if c > 0 { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( - value: 0xA0_0000 | ((c & 0x7F) as base.u64), + value: 0x40_0000 | ((c & 0x7F) as base.u64), length: 2) continue.string_loop @@ -286,7 +287,7 @@ // Not a Unicode surrogate. We're good. args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( - value: 0xA0_0000 | (uni4_value as base.u64), + value: 0x40_0000 | (uni4_value as base.u64), length: 6) continue.string_loop @@ -349,7 +350,7 @@ uni4_value -= 0xDC00 args.src.skip32_fast!(actual: 8, worst_case: 8) args.dst.write_fast_token!( - value: 0xA0_0000 | ((uni4_high_surrogate + uni4_value) as base.u64), + value: 0x40_0000 | ((uni4_high_surrogate + uni4_value) as base.u64), length: 12) continue.string_loop @@ -376,7 +377,7 @@ args.src.skip32_fast!(actual: 1, worst_case: 1) if string_length >= 0xFFFE { args.dst.write_fast_token!( - value: 0x20_0001, + value: 0x20_0221, length: 0xFFFF) string_length = 0 continue.string_loop @@ -446,9 +447,9 @@ { number_length = this.decode_number!(src: args.src) number_status = number_length >> 8 - token_value = 0x80_0004 + token_value = 0x60_0004 if (number_length & 0x80) <> 0 { - token_value = 0x80_0002 + token_value = 0x60_0002 } number_length = number_length & 0x7F if number_status == 0 { @@ -484,13 +485,13 @@ // -------- END parse numbers. } else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE. - token_value = 0x60_4011 + token_value = 0x80_4011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE. - token_value = 0x60_4041 + token_value = 0x80_4041 } else { - token_value = 0x60_4021 + token_value = 0x80_4021 } if depth >= 1024 { return "#unsupported recursion depth" @@ -512,7 +513,7 @@ args.src.skip32_fast!(actual: 1, worst_case: 1) if depth <= 1 { args.dst.write_fast_token!( - value: 0x60_1042, + value: 0x80_1042, length: 1) break.outer } @@ -521,14 +522,14 @@ stack_bit = (depth - 1) & 31 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( - value: 0x60_2042, + value: 0x80_2042, length: 1) // 0x0104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA). expect = 0x0104 expect_after_value = 0x0104 } else { args.dst.write_fast_token!( - value: 0x60_4042, + value: 0x80_4042, length: 1) // 0x0044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). expect = 0x0044 @@ -537,13 +538,13 @@ continue.outer } else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET. - token_value = 0x60_2011 + token_value = 0x80_2011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE. - token_value = 0x60_2041 + token_value = 0x80_2041 } else { - token_value = 0x60_2021 + token_value = 0x80_2021 } if depth >= 1024 { return "#unsupported recursion depth" @@ -565,7 +566,7 @@ args.src.skip32_fast!(actual: 1, worst_case: 1) if depth <= 1 { args.dst.write_fast_token!( - value: 0x60_1022, + value: 0x80_1022, length: 1) break.outer } @@ -574,14 +575,14 @@ stack_bit = (depth - 1) & 31 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( - value: 0x60_2022, + value: 0x80_2022, length: 1) // 0x0104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). expect = 0x0104 expect_after_value = 0x0104 } else { args.dst.write_fast_token!( - value: 0x60_4022, + value: 0x80_4022, length: 1) // 0x0044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA). expect = 0x0044 @@ -593,7 +594,7 @@ match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false". if match == 0 { args.dst.write_fast_token!( - value: 0x80_0401, + value: 0x60_0401, length: 5) if args.src.available() < 5 { return "#internal error: inconsistent I/O" @@ -609,7 +610,7 @@ match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true". if match == 0 { args.dst.write_fast_token!( - value: 0x80_0801, + value: 0x60_0801, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O" @@ -625,7 +626,7 @@ match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null". if match == 0 { args.dst.write_fast_token!( - value: 0x80_0201, + value: 0x60_0201, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O"