Add TOKEN__VBD__STRING__CHAIN_ETC
diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 2f91406..c4d3b1d 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h
@@ -82,15 +82,31 @@ // -------- -// "DEFINITELY_FOO" means that the destination bytes (and also the source -// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that -// the lack of the bit is conservative: it is valid for all-ASCII strings to -// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set. +// DEFINITELY_FOO means that the destination bytes (and also the source bytes, +// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack +// of the bit means "maybe FOO". It does not necessarily mean "not FOO". +// +// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII +// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes +// Unicode (UTF-8) strings from byte strings. MUST means that the the token +// producer (e.g. parser) must verify this. SHOULD means that the token +// consumer (e.g. renderer) should verify this. +// +// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII +// code points (with multi-byte UTF-8 encodings) do not straddle token +// boundaries. Checking UTF-8 validity can inspect each token separately. +// +// The lack of any particular bit is conservative: it is valid for all-ASCII +// strings, in a single- or multi-token chain, to have none of these bits set. #define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004 +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040 -// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly -// padded) produces multiples of D destination bytes. For example, +// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded) +// produces multiples of D destination bytes. For example, // CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where // 12 src bytes encode 3 dst bytes. // @@ -101,13 +117,13 @@ // When src is the empty string, multiple conversion algorithms are applicable // (so these bits are not necessarily mutually exclusive), all producing the // same empty dst string. -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000 // --------
diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index 514ad39..14335a7 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go
@@ -490,8 +490,9 @@ "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + "" + - "// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutual" + - "ly exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n" + + "// --------\n\n// DEFINITELY_FOO means that the destination bytes (and also the source bytes,\n// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack\n// of the bit means \"maybe FOO\". It does not necessarily mean \"not FOO\".\n//\n// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII\n// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes\n// Unicode (UTF-8) strings from byte strings. MUST means that the the token\n// producer (e.g. parser) must verify this. SHOULD means that the token\n// consumer (e.g. renderer) should verify this.\n//\n// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII\n// code points (with multi-byte UTF-8 encodings) do not straddle token\n// boundaries. Checking UTF-8 validity can inspect each token separately.\n//\n// The lack of any particular bit is conservative: it is valid for all-ASCII\n// strings, in a single- or multi-token chain, to have none of these bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING_" + + "_DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040\n\n// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)\n// produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WU" + + "FFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n" + "" +
diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index cc52d88..6944190 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go
@@ -101,15 +101,19 @@ // ---- {t.IDU32, "0x00001", "TOKEN__VBD__STRING__DEFINITELY_UTF_8"}, - {t.IDU32, "0x00002", "TOKEN__VBD__STRING__DEFINITELY_ASCII"}, + {t.IDU32, "0x00002", "TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8"}, + {t.IDU32, "0x00004", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8"}, + {t.IDU32, "0x00010", "TOKEN__VBD__STRING__DEFINITELY_ASCII"}, + {t.IDU32, "0x00020", "TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII"}, + {t.IDU32, "0x00040", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII"}, - {t.IDU32, "0x00010", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"}, - {t.IDU32, "0x00020", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"}, - {t.IDU32, "0x00040", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"}, - {t.IDU32, "0x00080", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"}, - {t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"}, - {t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"}, - {t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"}, + {t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"}, + {t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"}, + {t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"}, + {t.IDU32, "0x00800", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"}, + {t.IDU32, "0x01000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"}, + {t.IDU32, "0x02000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"}, + {t.IDU32, "0x04000", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"}, // ----
diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 7098044..9032742 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c
@@ -2141,15 +2141,31 @@ // -------- -// "DEFINITELY_FOO" means that the destination bytes (and also the source -// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that -// the lack of the bit is conservative: it is valid for all-ASCII strings to -// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set. +// DEFINITELY_FOO means that the destination bytes (and also the source bytes, +// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack +// of the bit means "maybe FOO". It does not necessarily mean "not FOO". +// +// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII +// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes +// Unicode (UTF-8) strings from byte strings. MUST means that the the token +// producer (e.g. parser) must verify this. SHOULD means that the token +// consumer (e.g. renderer) should verify this. +// +// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII +// code points (with multi-byte UTF-8 encodings) do not straddle token +// boundaries. Checking UTF-8 validity can inspect each token separately. +// +// The lack of any particular bit is conservative: it is valid for all-ASCII +// strings, in a single- or multi-token chain, to have none of these bits set. #define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004 +#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040 -// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly -// padded) produces multiples of D destination bytes. For example, +// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded) +// produces multiples of D destination bytes. For example, // CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where // 12 src bytes encode 3 dst bytes. // @@ -2160,13 +2176,13 @@ // When src is the empty string, multiple conversion algorithms are applicable // (so these bits are not necessarily mutually exclusive), all producing the // same empty dst string. -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200 -#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000 // -------- @@ -23102,7 +23118,7 @@ } if (v_class == 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -23119,7 +23135,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23144,7 +23160,7 @@ (iop_a_src += 4, wuffs_base__make_empty_struct()); if (v_string_length > 65527) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23159,7 +23175,7 @@ (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_string_length >= 65531) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23170,7 +23186,7 @@ } else if (v_char == 1) { if (v_string_length != 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23179,7 +23195,7 @@ } else if (v_char == 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23405,7 +23421,7 @@ goto exit; } *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194432)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4196355)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; @@ -23416,7 +23432,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) < 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23448,7 +23464,7 @@ (iop_a_src += 2, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23461,7 +23477,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) < 3) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23494,7 +23510,7 @@ (iop_a_src += 3, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23508,7 +23524,7 @@ if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23544,7 +23560,7 @@ (iop_a_src += 4, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23557,7 +23573,7 @@ } if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -23608,7 +23624,7 @@ } (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__2__break; }
diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 260ec9c..7407994 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs
@@ -206,6 +206,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__DEFINITELY_ASCII | base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, continued: 1, @@ -228,6 +229,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -260,6 +262,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 4) @@ -279,6 +282,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: 0xFFFC) @@ -294,6 +298,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -307,6 +312,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -594,6 +600,8 @@ args.dst.write_simple_token_fast!( value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | + base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X, continued: 1, length: backslash_x_length) @@ -610,6 +618,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -646,6 +655,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 2) @@ -663,6 +673,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -703,6 +714,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 3) @@ -721,6 +733,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -760,6 +773,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 4) @@ -777,6 +791,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -830,6 +845,7 @@ value_major: 0, value_minor: (base.TOKEN__VBC__STRING << 21) | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__DEFINITELY_ASCII | base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, continued: 0,
diff --git a/std/json/decode_quirks.wuffs b/std/json/decode_quirks.wuffs index cc03ed6..946ab78 100644 --- a/std/json/decode_quirks.wuffs +++ b/std/json/decode_quirks.wuffs
@@ -81,17 +81,24 @@ // "\x", "\x9", "\x9$" and "\X99" are all still rejected. // // It is up to the caller how to interpret the two hexadecimal digits. There -// are two sensible choices: as a byte or as a Unicode code point. +// are two sensible choices: as-a-byte or as-a-Unicode-code-point. // -// As a byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. +// As-a-byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. // Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and // even if the overall byte string is valid UTF-8, tokens are also no longer // guaranteed to split on UTF-8 boundaries. // -// As a Unicode code point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3, +// As-a-Unicode-code-point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3, // 0xAF and 0x7A. The UTF-8 encoding of U+00EF LATIN SMALL LETTER I WITH // DIAERESIS is the two byte sequence (0xC3, 0xAF). Decoded strings are still // valid UTF-8 and tokens still split on UTF-8 boundaries. +// +// The parser cannot know which interpretation the caller will choose, so it +// will still mark its string tokens with TOKEN__VBD__STRING__DEFINITELY_UTF_8 +// and TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8, like all other JSON string +// tokens, even though the as-a-byte interpretation can result in invalid +// UTF-8. If the caller chooses as-a-byte, it is up to the caller whether and +// how to run UTF-8 validation. pub const QUIRK_ALLOW_BACKSLASH_X : base.u32 = 0x4909_9400 | 0x08 // When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string,
diff --git a/test/c/std/json.c b/test/c/std/json.c index 3f3784e..d586d43 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c
@@ -2452,7 +2452,7 @@ uint64_t vbd = wuffs_base__token__value_base_detail(t); uint64_t token_length = wuffs_base__token__length(t); if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) && - (vbd == + (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) { uint8_t b[8] = {0}; size_t n = wuffs_base__hexadecimal__decode4(
diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens index 1750ef5..8eba91c 100644 --- a/test/data/australian-abc-local-stations.tokens +++ b/test/data/australian-abc-local-stations.tokens Binary files differ
diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens index d2518f2..7569ad1 100644 --- a/test/data/json-quirks.tokens +++ b/test/data/json-quirks.tokens Binary files differ
diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens index 7bab270..1fd7fb8 100644 --- a/test/data/json-things.unformatted.tokens +++ b/test/data/json-things.unformatted.tokens Binary files differ