Add WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_ETC
diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h
index 9b86324..31f02e7 100644
--- a/internal/cgen/base/token-public.h
+++ b/internal/cgen/base/token-public.h
@@ -169,16 +169,23 @@
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004
+
+// The number 300 might be represented as "\x01\x2C", "\x2C\x01\x00\x00" or
+// "300", which are big-endian, little-endian or text. For binary formats, the
+// token length discriminates e.g. u16 little-endian vs u32 little-endian.
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400
// --------
diff --git a/internal/cgen/data.go b/internal/cgen/data.go
index 505e252..9d9a653 100644
--- a/internal/cgen/data.go
+++ b/internal/cgen/data.go
@@ -365,7 +365,8 @@
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n " +
"" +
- "// --------\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004\n\n" +
+ "// --------\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004\n\n// The number 300 might be represented as \"\\x01\\x2C\", \"\\x2C\\x01\\x00\\x00\" or\n// \"300\", which are big-endian, little-endian or text. For binary formats, the\n// token length discriminates e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_END" +
+ "IAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n" +
"" +
"// --------\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETA" +
"IL__MASK;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__to" +
diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c
index d20542a..67bf445 100644
--- a/release/c/wuffs-unsupported-snapshot.c
+++ b/release/c/wuffs-unsupported-snapshot.c
@@ -1886,16 +1886,23 @@
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
-// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.
+// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002
-#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004
+
+// The number 300 might be represented as "\x01\x2C", "\x2C\x01\x00\x00" or
+// "300", which are big-endian, little-endian or text. For binary formats, the
+// token length discriminates e.g. u16 little-endian vs u32 little-endian.
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200
+#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400
// --------
@@ -20814,9 +20821,9 @@
iop_a_src = a_src->data.ptr + a_src->meta.ri;
}
v_number_status = (v_number_length >> 8);
- v_vminor = 10485763;
+ v_vminor = 10486787;
if ((v_number_length & 128) != 0) {
- v_vminor = 10485761;
+ v_vminor = 10486785;
}
v_number_length = (v_number_length & 127);
if (v_number_status == 0) {
diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs
index 00bec3b..194cfcb 100644
--- a/std/json/decode_json.wuffs
+++ b/std/json/decode_json.wuffs
@@ -609,9 +609,9 @@
{
number_length = this.decode_number!(src: args.src)
number_status = number_length >> 8
- vminor = 0xA0_0003
+ vminor = 0xA0_0403
if (number_length & 0x80) <> 0 {
- vminor = 0xA0_0001
+ vminor = 0xA0_0401
}
number_length = number_length & 0x7F
if number_status == 0 {
diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens
index 0ba8ba7..4274df8 100644
--- a/test/data/australian-abc-local-stations.tokens
+++ b/test/data/australian-abc-local-stations.tokens
Binary files differ
diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens
index e88cf8c..8fa0137 100644
--- a/test/data/json-things.unformatted.tokens
+++ b/test/data/json-things.unformatted.tokens
Binary files differ
