Remove wuffs_base__token__link_prev
diff --git a/doc/note/tokens.md b/doc/note/tokens.md
index 8a9bde0..5577f2d 100644
--- a/doc/note/tokens.md
+++ b/doc/note/tokens.md
@@ -36,25 +36,22 @@
A token is just a `uint64_t`. The broad divisions are:
-- Bits 63 .. 18 (46 bits) is the value.
-- Bits 17 .. 16 (2 bits) is `LP` and `LN` (`link_prev` and `link_next`).
+- Bits 63 .. 17 (47 bits) is the value.
+- Bits 16 .. 16 (1 bit) is the `continued` bit.
- Bits 15 .. 0 (16 bits) is the length.
-The `LP` and `LN` bits denote tokens that are part of a multi-token chain:
-
-- `LP` means that this token is not the first (there is a previous token).
-- `LN` means that this token is not the last (there is a next token).
-
-A stand-alone token will have both link bits set to zero.
+The `continued` bit is whether that the token chain for this token also
+contains the next token. The final token in a token chain, including
+stand-alone tokens, will have the `continued` bit set to zero.
```
-+-----+-------------+-------+-------------+-----+-----+-----------+
-| 1 | 21 | 3 | 21 | 1 | 1 | 16 |
-+-----+-------------+-------+-------------+-----+-----+-----------+
-[..................value..................] LP LN length
-[..1..|..........~value_extension.........]
-[..0..|.value_major.|.....value_minor.....]
-[..0..|......0......|..VBC..|.....VBD.....]
++-----+-------------+-------+---------------+-----+-----------+
+| 1 | 21 | 3 | 22 | 1 | 16 |
++-----+-------------+-------+---------------+-----+-----------+
+[...................value...................] con length
+[..1..|...........~value_extension..........]
+[..0..|.value_major.|......value_minor......]
+[..0..|......0......|..VBC..|......VBD......]
```
The value bits can be sub-divided in multiple ways. First, the high bit:
@@ -64,7 +61,7 @@
### Extended Tokens
-- Bits 62 .. 18 (45 bits) is the bitwise-not (~) of the `value_extension`.
+- Bits 62 .. 17 (46 bits) is the bitwise-not (~) of the `value_extension`.
Extended tokens are typically part of a multi-token chain whose first token is
a simple token that provides the semantics for each `value_extension`.
@@ -73,7 +70,7 @@
### Simple Tokens
- Bits 62 .. 42 (21 bits) is the `value_major`.
-- Bits 41 .. 18 (24 bits) is the `value_minor`.
+- Bits 41 .. 17 (25 bits) is the `value_minor`.
The `value_major` is a 21-bit [Base38](doc/note/base38-and-fourcc.md) number.
For example, an HTML tokenizer might produce a combination of "base" tokens
@@ -90,13 +87,13 @@
A zero `value_major` is reserved for Wuffs' built-in "base" package. The
`value_minor` is further sub-divided:
- - Bits 41 .. 39 (3 bits) is the `VBC` (`value_base_category`).
- - Bits 38 .. 18 (21 bits) is the `VBD` (`value_base_detail`).
+- Bits 41 .. 38 (4 bits) is the `VBC` (`value_base_category`).
+- Bits 37 .. 17 (21 bits) is the `VBD` (`value_base_detail`).
-The high 46 bits (bits 63 .. 18) only have `VBC` and `VBD` semantics when the
+The high 47 bits (bits 63 .. 17) only have `VBC` and `VBD` semantics when the
high 22 bits (the `extended` and `value_major` parts) are all zero. An
-equivalent test is that the high 25 bits (the notional `VBC`), as either an
-unsigned integer or a sign-extended integer, is in the range `0 ..= 7`.
+equivalent test is that the high 26 bits (the notional `VBC`), as either an
+unsigned integer or a sign-extended integer, is in the range `0 ..= 15`.
These `VBC`s organize tokens into broad groups, generally applicable (as
opposed to being e.g. HTML-specific or JSON-specific). For example, strings and
@@ -142,15 +139,17 @@
## Example Token Stream
```
-$ gcc script/print-json-token-debug-format.c && \
-> ./a.out -all-tokens -human-readable < test/data/json-things.formatted.json
-pos=0x00000000 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x004011
-pos=0x00000001 len=0x0005 link=0b00 vbc=0:Filler........... vbd=0x000000
-pos=0x00000006 len=0x0001 link=0b01 vbc=2:String........... vbd=0x000013
-pos=0x00000007 len=0x0002 link=0b11 vbc=2:String........... vbd=0x000021
-pos=0x00000009 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013
+$ gcc script/print-json-token-debug-format.c -o pjtdf
+$ ./pjtdf -all-tokens -human-readable < test/data/json-things.formatted.json
+pos=0x00000000 len=0x0001 con=0 vbc=1:Structure........ vbd=0x004011
+pos=0x00000001 len=0x0005 con=0 vbc=0:Filler........... vbd=0x000000
+pos=0x00000006 len=0x0001 con=1 vbc=2:String........... vbd=0x000013
+pos=0x00000007 len=0x0002 con=1 vbc=2:String........... vbd=0x000021
+pos=0x00000009 len=0x0001 con=0 vbc=2:String........... vbd=0x000013
etc
-pos=0x00000094 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013
-pos=0x00000095 len=0x0001 link=0b00 vbc=0:Filler........... vbd=0x000000
-pos=0x00000096 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x001042
+pos=0x00000090 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A
+pos=0x00000092 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A
+pos=0x00000094 len=0x0001 con=0 vbc=2:String........... vbd=0x000013
+pos=0x00000095 len=0x0001 con=0 vbc=0:Filler........... vbd=0x000000
+pos=0x00000096 len=0x0001 con=0 vbc=1:Structure........ vbd=0x001042
```
diff --git a/example/jsonfindptrs/jsonfindptrs.cc b/example/jsonfindptrs/jsonfindptrs.cc
index 2bf74f9..5c5816f 100644
--- a/example/jsonfindptrs/jsonfindptrs.cc
+++ b/example/jsonfindptrs/jsonfindptrs.cc
@@ -674,7 +674,7 @@
return Result("main: internal error: unexpected token");
}
- if (!tsr.token.link_next()) {
+ if (!tsr.token.continued()) {
break;
}
tsr = ts.next();
diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc
index e4aa8a7..7c3dc2c 100644
--- a/example/jsonptr/jsonptr.cc
+++ b/example/jsonptr/jsonptr.cc
@@ -979,15 +979,15 @@
return "main: internal error: unexpected string-token conversion";
}
- if (t.link_next()) {
+ if (t.continued()) {
return nullptr;
}
TRY(write_dst("\"", 1));
goto after_value;
case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT:
- if (!t.link_next()) {
- return "main: internal error: unexpected unlinked token";
+ if (!t.continued()) {
+ return "main: internal error: unexpected non-continued UCP token";
}
TRY(handle_unicode_code_point(vbd));
g_query.incremental_match_code_point(vbd);
@@ -1049,12 +1049,12 @@
// Skip filler tokens (e.g. whitespace).
if (t.value() == 0) {
- start_of_token_chain = !t.link_next();
+ start_of_token_chain = !t.continued();
continue;
}
const char* z = handle_token(t, start_of_token_chain);
- start_of_token_chain = !t.link_next();
+ start_of_token_chain = !t.continued();
if (z == nullptr) {
continue;
} else if (z == g_eod) {
diff --git a/fuzz/c/std/json_fuzzer.c b/fuzz/c/std/json_fuzzer.c
index c3ae16e..916237c 100644
--- a/fuzz/c/std/json_fuzzer.c
+++ b/fuzz/c/std/json_fuzzer.c
@@ -196,7 +196,7 @@
// After a complete JSON value, update the parity (even/odd count) of the
// container.
- if (!wuffs_base__token__link_next(&t) &&
+ if (!wuffs_base__token__continued(&t) &&
(vbc != WUFFS_BASE__TOKEN__VBC__FILLER) &&
((vbc != WUFFS_BASE__TOKEN__VBC__STRUCTURE) ||
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP))) {
@@ -368,8 +368,8 @@
if (depth != 0) {
return "fuzz: internal error: decoded OK but final depth was not zero";
- } else if (wuffs_base__token__link_next(&final_token)) {
- return "fuzz: internal error: decoded OK but final token had link_next";
+ } else if (wuffs_base__token__continued(&final_token)) {
+ return "fuzz: internal error: decoded OK but final token was continued";
}
return NULL;
}
diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h
index aa37f13..eb05a76 100644
--- a/internal/cgen/base/token-public.h
+++ b/internal/cgen/base/token-public.h
@@ -29,8 +29,7 @@
inline int64_t value_base_category() const;
inline uint64_t value_minor() const;
inline uint64_t value_base_detail() const;
- inline bool link_prev() const;
- inline bool link_next() const;
+ inline bool continued() const;
inline uint64_t length() const;
#endif // __cplusplus
@@ -47,18 +46,15 @@
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
-#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
-#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18
+#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17
+#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
-#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
-#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
-#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
-#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
+#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17
+#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38
+#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17
+#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
-#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
-#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
-
// --------
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
@@ -179,7 +175,7 @@
static inline uint64_t //
wuffs_base__token__value_minor(const wuffs_base__token* t) {
- return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;
+ return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;
}
static inline uint64_t //
@@ -188,13 +184,8 @@
}
static inline bool //
-wuffs_base__token__link_prev(const wuffs_base__token* t) {
- return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
-}
-
-static inline bool //
-wuffs_base__token__link_next(const wuffs_base__token* t) {
- return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
+wuffs_base__token__continued(const wuffs_base__token* t) {
+ return t->repr & 0x10000;
}
static inline uint64_t //
@@ -235,13 +226,8 @@
}
inline bool //
-wuffs_base__token::link_prev() const {
- return wuffs_base__token__link_prev(this);
-}
-
-inline bool //
-wuffs_base__token::link_next() const {
- return wuffs_base__token__link_next(this);
+wuffs_base__token::continued() const {
+ return wuffs_base__token__continued(this);
}
inline uint64_t //
diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go
index 197c7f3..5eca912 100644
--- a/internal/cgen/builtin.go
+++ b/internal/cgen/builtin.go
@@ -350,7 +350,7 @@
if err := g.writeExpr(b, args[2].AsArg().Value(), depth); err != nil {
return err
}
- b.writes(")) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(")
+ b.writes(")) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(")
}
if err := g.writeExpr(b, args[3].AsArg().Value(), depth); err != nil {
diff --git a/internal/cgen/data.go b/internal/cgen/data.go
index 7d5d0e6..82a3973 100644
--- a/internal/cgen/data.go
+++ b/internal/cgen/data.go
@@ -382,9 +382,9 @@
""
const baseTokenPublicH = "" +
- "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
+ "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
"" +
- "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
+ "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n " +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " +
"" +
@@ -401,8 +401,8 @@
"ttle-endian or text. For binary formats, the\n// token length discriminates e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n" +
"" +
"// --------\n\n// wuffs_base__token__value returns the token's high 46 bits, sign-extended. A\n// negative value means an extended token, non-negative means a simple token.\nstatic inline int64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE__SHIFT;\n}\n\n// wuffs_base__token__value_extension returns a negative value if the token was\n// not an extended token.\nstatic inline int64_t //\nwuffs_base__token__value_extension(const wuffs_base__token* t) {\n return (~(int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT;\n}\n\n// wuffs_base__token__value_major returns a negative value if the token was not\n// a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT;\n}\n\n// wuffs_base__token__value_base_category returns a negative value if the token\n// was not a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_base_cat" +
- "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base_" +
- "_token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::link_prev() const {\n return wuffs_base__token__link_prev(this);\n}\n\ninline bool //\nwuffs_base__token::link_next() const {\n return wuffs_base__token__link_next(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
+ "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n retu" +
+ "rn wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"" +
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
"" +
diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go
index 1bd52cc..34ba6ec 100644
--- a/lang/builtin/builtin.go
+++ b/lang/builtin/builtin.go
@@ -332,8 +332,8 @@
// ---- token_writer
"token_writer.write_simple_token_fast!(" +
- "value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0xFF_FFFF]," +
- "link: u32[..= 0x3], length: u32[..= 0xFFFF])",
+ "value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0x1FF_FFFF]," +
+ "continued: u32[..= 0x1], length: u32[..= 0xFFFF])",
"token_writer.available() u64",
diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c
index 9b0b950..ed2198f 100644
--- a/release/c/wuffs-unsupported-snapshot.c
+++ b/release/c/wuffs-unsupported-snapshot.c
@@ -1888,8 +1888,7 @@
inline int64_t value_base_category() const;
inline uint64_t value_minor() const;
inline uint64_t value_base_detail() const;
- inline bool link_prev() const;
- inline bool link_next() const;
+ inline bool continued() const;
inline uint64_t length() const;
#endif // __cplusplus
@@ -1906,18 +1905,15 @@
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
-#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
-#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18
+#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17
+#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
-#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
-#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
-#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
-#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
+#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17
+#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38
+#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17
+#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
-#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
-#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
-
// --------
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
@@ -2038,7 +2034,7 @@
static inline uint64_t //
wuffs_base__token__value_minor(const wuffs_base__token* t) {
- return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;
+ return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;
}
static inline uint64_t //
@@ -2047,13 +2043,8 @@
}
static inline bool //
-wuffs_base__token__link_prev(const wuffs_base__token* t) {
- return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
-}
-
-static inline bool //
-wuffs_base__token__link_next(const wuffs_base__token* t) {
- return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
+wuffs_base__token__continued(const wuffs_base__token* t) {
+ return t->repr & 0x10000;
}
static inline uint64_t //
@@ -2094,13 +2085,8 @@
}
inline bool //
-wuffs_base__token::link_prev() const {
- return wuffs_base__token__link_prev(this);
-}
-
-inline bool //
-wuffs_base__token::link_next() const {
- return wuffs_base__token__link_next(this);
+wuffs_base__token::continued() const {
+ return wuffs_base__token__continued(this);
}
inline uint64_t //
@@ -6135,9 +6121,6 @@
uint32_t v_expect_after_value;
} s_decode_tokens[1];
struct {
- uint32_t v_link_prev;
- } s_decode_comment[1];
- struct {
uint32_t v_neg;
} s_decode_inf_nan[1];
} private_data;
@@ -21233,7 +21216,7 @@
if (v_class == 1) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
label__string_loop_outer__continue:;
@@ -21252,7 +21235,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21281,7 +21264,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 4)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21298,7 +21281,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(65532))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21311,7 +21294,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21322,7 +21305,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21352,7 +21335,7 @@
(((uint64_t)(
(6291456 | ((uint32_t)((v_backslash & 127))))))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
} else if (v_backslash != 0) {
@@ -21365,7 +21348,8 @@
((uint32_t)(wuffs_json__lut_quirky_backslashes[(
v_backslash & 7)])))))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
@@ -21411,7 +21395,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((6291456 | v_uni4_value)))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
} else if (v_uni4_value >= 56320) {
@@ -21424,8 +21409,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3))
- << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(6))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
@@ -21481,7 +21466,8 @@
(((uint64_t)((6291456 | v_uni4_high_surrogate |
v_uni4_value)))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(12))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
@@ -21498,7 +21484,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
@@ -21562,7 +21549,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((6291456 | (v_uni8_value & 2097151))))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
} else if (self->private_impl
@@ -21571,7 +21559,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
}
@@ -21619,7 +21608,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194432))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_backslash_x_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
@@ -21633,7 +21622,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21647,7 +21637,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
@@ -21674,7 +21665,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 2)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21689,7 +21681,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21703,7 +21696,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
@@ -21734,7 +21728,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 3)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21750,7 +21745,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21764,7 +21760,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
@@ -21795,7 +21792,8 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1))
+ << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 4)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21810,7 +21808,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -21824,7 +21822,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((6291456 | ((uint32_t)((v_char & 127))))))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_outer__continue;
@@ -21837,7 +21835,7 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6356989))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
goto label__string_loop_outer__continue;
@@ -21869,7 +21867,6 @@
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194323))
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__2__break;
}
@@ -22566,7 +22563,6 @@
uint8_t v_c = 0;
uint16_t v_c2 = 0;
- uint32_t v_link_prev = 0;
uint32_t v_length = 0;
wuffs_base__token* iop_a_dst = NULL;
@@ -22594,9 +22590,6 @@
}
uint32_t coro_susp_point = self->private_impl.p_decode_comment[0];
- if (coro_susp_point) {
- v_link_prev = self->private_data.s_decode_comment[0].v_link_prev;
- }
switch (coro_susp_point) {
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
@@ -22633,10 +22626,8 @@
if (v_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)((1 | v_link_prev)))
- << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
- v_link_prev = 2;
}
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
@@ -22653,7 +22644,6 @@
(iop_a_src += 2, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 2)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
status = wuffs_base__make_status(NULL);
@@ -22663,12 +22653,10 @@
if (v_length >= 65533) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)((1 | v_link_prev)))
- << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_length = 0;
- v_link_prev = 2;
goto label__comment_block__continue;
}
v_length += 1;
@@ -22691,10 +22679,8 @@
if (v_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)((1 | v_link_prev)))
- << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
- v_link_prev = 2;
}
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
@@ -22711,7 +22697,6 @@
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
status = wuffs_base__make_status(NULL);
@@ -22721,12 +22706,10 @@
if (v_length >= 65533) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
- (((uint64_t)((1 | v_link_prev)))
- << WUFFS_BASE__TOKEN__LINK__SHIFT) |
+ (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_length + 1)))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_length = 0;
- v_link_prev = 2;
goto label__comment_line__continue;
}
v_length += 1;
@@ -22745,7 +22728,6 @@
suspend:
self->private_impl.p_decode_comment[0] =
wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
- self->private_data.s_decode_comment[0].v_link_prev = v_link_prev;
goto exit;
exit:
diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c
index 89877b9..e8a81bc 100644
--- a/script/print-json-token-debug-format.c
+++ b/script/print-json-token-debug-format.c
@@ -24,16 +24,15 @@
// It prints 16 bytes (128 bits) per token, containing big-endian numbers:
//
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
-// | | | | | | VALUE_EXTENSION |
-// | POS | LEN | LP| LN|EXT|VALUE_MAJOR| VALUE_MINOR |
-// | | | | | | 0 |VBC| VBD |
+// | | | | | VALUE_EXTENSION |
+// | POS | LEN | CON |EXT|VALUE_MAJOR| VALUE_MINOR |
+// | | | | | 0 |VBC| VBD |
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
//
// - POS (4 bytes) is the position: the sum of all previous tokens' lengths,
// including elided tokens.
// - LEN (2 bytes) is the length.
-// - LP (1 bytes) is the link_prev bit.
-// - LN (1 bytes) is the link_next bit
+// - CON (2 bytes) is the continued bit
// - EXT (1 bytes) is 1 for extended and 0 for simple tokens.
//
// Extended tokens have a VALUE_EXTENSION (7 bytes).
@@ -54,8 +53,8 @@
// If the input or output is larger than the program's buffers (64 MiB and
// 131072 tokens by default), there may be multiple valid tokenizations of any
// given input. For example, if a source string "abcde" straddles an I/O
-// boundary, it may be tokenized as single (no-link) 5-length string or as a
-// 3-length link_next'ed string followed by a 2-length link_prev'ed string.
+// boundary, it may be tokenized as single (not continued) 5-length string or
+// as a 3-length continued string followed by a 2-length string.
//
// A Wuffs token stream, in general, can support inputs more than 0xFFFF_FFFF
// bytes long, but this program can not, as it tracks the tokens' cumulative
@@ -202,7 +201,7 @@
return NULL;
}
-const char* g_vbc_names[8] = {
+const char* g_vbc_names[16] = {
"0:Filler..........", //
"1:Structure.......", //
"2:String..........", //
@@ -211,6 +210,14 @@
"5:Number..........", //
"6:Reserved........", //
"7:Reserved........", //
+ "8:Reserved........", //
+ "9:Reserved........", //
+ "A:Reserved........", //
+ "B:Reserved........", //
+ "C:Reserved........", //
+ "D:Reserved........", //
+ "E:Reserved........", //
+ "F:Reserved........", //
};
const int g_base38_decode[38] = {
@@ -277,16 +284,15 @@
uint16_t len = wuffs_base__token__length(t);
if (g_flags.all_tokens || (wuffs_base__token__value(t) != 0)) {
- uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
- uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
+ uint16_t con = wuffs_base__token__continued(t) ? 1 : 0;
int32_t vmajor = wuffs_base__token__value_major(t);
uint32_t vminor = wuffs_base__token__value_minor(t);
uint8_t vbc = wuffs_base__token__value_base_category(t);
uint32_t vbd = wuffs_base__token__value_base_detail(t);
if (g_flags.human_readable) {
- printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " link=0b%d%d ",
- (uint32_t)(pos), len, (int)(lp), (int)(ln));
+ printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " con=%" PRId16 " ",
+ (uint32_t)(pos), len, con);
if (vmajor > 0) {
uint32_t m = vmajor;
@@ -302,7 +308,8 @@
vmajor, g_base38_decode[m0], g_base38_decode[m1],
g_base38_decode[m2], g_base38_decode[m3], vminor);
} else if (vmajor == 0) {
- printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 7], vbd);
+ printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15],
+ vbd);
} else {
printf("extended... vextension=0x%012" PRIX64 "\n",
wuffs_base__token__value_extension(t));
@@ -312,8 +319,7 @@
uint8_t buf[16];
wuffs_base__store_u32be__no_bounds_check(&buf[0x0], (uint32_t)(pos));
wuffs_base__store_u16be__no_bounds_check(&buf[0x4], len);
- wuffs_base__store_u8__no_bounds_check(&buf[0x0006], lp);
- wuffs_base__store_u8__no_bounds_check(&buf[0x0007], ln);
+ wuffs_base__store_u16be__no_bounds_check(&buf[0x6], con);
if (vmajor > 0) {
wuffs_base__store_u32be__no_bounds_check(&buf[0x8], vmajor);
wuffs_base__store_u32be__no_bounds_check(&buf[0xC], vminor);
diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs
index c5169d3..44f78e0 100644
--- a/std/json/decode_json.wuffs
+++ b/std/json/decode_json.wuffs
@@ -188,7 +188,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0,
- link: 0x0,
+ continued: 0,
length: whitespace_length)
whitespace_length = 0
}
@@ -211,7 +211,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0,
- link: 0x0,
+ continued: 0,
length: 0xFFFF)
whitespace_length = 0
continue.outer
@@ -224,7 +224,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0,
- link: 0x0,
+ continued: 0,
length: whitespace_length)
whitespace_length = 0
if args.dst.available() <= 0 {
@@ -250,7 +250,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0013,
- link: 0x1,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
@@ -269,7 +269,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
}
@@ -299,7 +299,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length + 4)
string_length = 0
continue.string_loop_outer
@@ -316,7 +316,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: 0xFFFC)
string_length = 0
continue.string_loop_outer
@@ -329,7 +329,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
}
@@ -340,7 +340,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
@@ -365,7 +365,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | ((backslash & 0x7F) as base.u32),
- link: 0x3,
+ continued: 1,
length: 2)
continue.string_loop_outer
@@ -375,7 +375,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32),
- link: 0x3,
+ continued: 1,
length: 2)
continue.string_loop_outer
}
@@ -419,7 +419,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | uni4_value,
- link: 0x3,
+ continued: 1,
length: 6)
continue.string_loop_outer
@@ -440,7 +440,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 6)
continue.string_loop_outer
}
@@ -491,7 +491,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | uni4_high_surrogate | uni4_value,
- link: 0x3,
+ continued: 1,
length: 12)
continue.string_loop_outer
}
@@ -505,7 +505,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 6)
continue.string_loop_outer
}
@@ -563,7 +563,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | (uni8_value & 0x1F_FFFF),
- link: 0x3,
+ continued: 1,
length: 10)
continue.string_loop_outer
} else if this.quirk_enabled_replace_invalid_unicode {
@@ -571,7 +571,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 10)
continue.string_loop_outer
}
@@ -618,7 +618,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0080,
- link: 0x3,
+ continued: 1,
length: backslash_x_length)
continue.string_loop_outer
// -------- END backslash-x
@@ -632,7 +632,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
@@ -644,7 +644,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
@@ -665,7 +665,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length + 2)
string_length = 0
continue.string_loop_outer
@@ -680,7 +680,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
@@ -692,7 +692,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
@@ -717,7 +717,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length + 3)
string_length = 0
continue.string_loop_outer
@@ -733,7 +733,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
@@ -745,7 +745,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
@@ -769,7 +769,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length + 4)
string_length = 0
continue.string_loop_outer
@@ -784,7 +784,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0021,
- link: 0x3,
+ continued: 1,
length: string_length)
string_length = 0
if args.dst.available() <= 0 {
@@ -796,7 +796,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_0000 | ((char & 0x7F) as base.u32),
- link: 0x3,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
@@ -807,7 +807,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x60_FFFD,
- link: 0x3,
+ continued: 1,
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
continue.string_loop_outer
@@ -833,7 +833,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x40_0013,
- link: 0x2,
+ continued: 0,
length: 1)
break
} endwhile
@@ -854,7 +854,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0,
- link: 0x0,
+ continued: 0,
length: 1)
// What's valid after a comma depends on whether or not we're in an
// array or an object.
@@ -879,7 +879,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0,
- link: 0x0,
+ continued: 0,
length: 1)
expect = 0x1EB2 // 0x1EB2 is EXPECT_VALUE.
continue.outer
@@ -900,7 +900,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: vminor,
- link: 0x0,
+ continued: 0,
length: number_length)
break
}
@@ -955,7 +955,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: vminor,
- link: 0x0,
+ continued: 0,
length: 1)
expect = 0x1042 // 0x1042 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING).
expect_after_value = 0x1044 // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
@@ -967,7 +967,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_1042,
- link: 0x0,
+ continued: 0,
length: 1)
break.outer
}
@@ -978,7 +978,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_2042,
- link: 0x0,
+ continued: 0,
length: 1)
// 0x1104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA).
expect = 0x1104
@@ -987,7 +987,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_4042,
- link: 0x0,
+ continued: 0,
length: 1)
// 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
expect = 0x1044
@@ -1016,7 +1016,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: vminor,
- link: 0x0,
+ continued: 0,
length: 1)
expect = 0x1FB2 // 0x1FB2 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE).
expect_after_value = 0x1104 // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
@@ -1028,7 +1028,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_1022,
- link: 0x0,
+ continued: 0,
length: 1)
break.outer
}
@@ -1039,7 +1039,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_2022,
- link: 0x0,
+ continued: 0,
length: 1)
// 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
expect = 0x1104
@@ -1048,7 +1048,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x20_4022,
- link: 0x0,
+ continued: 0,
length: 1)
// 0x1044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA).
expect = 0x1044
@@ -1062,7 +1062,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x80_0004,
- link: 0x0,
+ continued: 0,
length: 5)
if args.src.available() < 5 {
return "#internal error: inconsistent I/O"
@@ -1080,7 +1080,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x80_0008,
- link: 0x0,
+ continued: 0,
length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"
@@ -1098,7 +1098,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x80_0002,
- link: 0x0,
+ continued: 0,
length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"
@@ -1339,7 +1339,7 @@
this.allow_leading_ars = false
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_simple_token_fast!(
- value_major: 0, value_minor: 0, link: 0x0, length: 1)
+ value_major: 0, value_minor: 0, continued: 0, length: 1)
continue
} else if (c == 0xEF) and this.allow_leading_ubom {
if args.src.available() < 3 {
@@ -1354,7 +1354,7 @@
this.allow_leading_ubom = false
args.src.skip32_fast!(actual: 3, worst_case: 3)
args.dst.write_simple_token_fast!(
- value_major: 0, value_minor: 0, link: 0x0, length: 3)
+ value_major: 0, value_minor: 0, continued: 0, length: 3)
continue
}
}
@@ -1363,10 +1363,9 @@
}
pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
- var c : base.u8
- var c2 : base.u16
- var link_prev : base.u32[..= 0x2]
- var length : base.u32[..= 0xFFFD]
+ var c : base.u8
+ var c2 : base.u16
+ var length : base.u32[..= 0xFFFD]
while (args.dst.available() <= 0) or (args.src.available() <= 1),
post args.dst.available() > 0,
@@ -1402,9 +1401,8 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x2,
- link: 0x1 | link_prev,
+ continued: 1,
length: length)
- link_prev = 0x2
}
if args.src.is_closed() {
return "#bad input"
@@ -1420,7 +1418,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x2,
- link: link_prev,
+ continued: 0,
length: length + 2)
return ok
}
@@ -1430,10 +1428,9 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x2,
- link: 0x1 | link_prev,
+ continued: 1,
length: length + 1)
length = 0
- link_prev = 0x2
continue.comment_block
}
length += 1
@@ -1459,9 +1456,8 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x1,
- link: 0x1 | link_prev,
+ continued: 1,
length: length)
- link_prev = 0x2
}
if args.src.is_closed() {
return "#bad input"
@@ -1477,7 +1473,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x1,
- link: link_prev,
+ continued: 0,
length: length + 1)
return ok
}
@@ -1487,10 +1483,9 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0x1,
- link: 0x1 | link_prev,
+ continued: 1,
length: length + 1)
length = 0
- link_prev = 0x2
continue.comment_line
}
length += 1
@@ -1527,7 +1522,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0020,
- link: 0x0,
+ continued: 0,
length: 8)
args.src.skip32_fast!(actual: 8, worst_case: 8)
return ok
@@ -1539,7 +1534,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0020,
- link: 0x0,
+ continued: 0,
length: 3)
args.src.skip32_fast!(actual: 3, worst_case: 3)
return ok
@@ -1548,7 +1543,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0080,
- link: 0x0,
+ continued: 0,
length: 3)
args.src.skip32_fast!(actual: 3, worst_case: 3)
return ok
@@ -1575,7 +1570,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg),
- link: 0x0,
+ continued: 0,
length: 9)
args.src.skip32_fast!(actual: 9, worst_case: 9)
return ok
@@ -1587,7 +1582,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg),
- link: 0x0,
+ continued: 0,
length: 4)
args.src.skip32_fast!(actual: 4, worst_case: 4)
return ok
@@ -1596,7 +1591,7 @@
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: 0xA0_0000 | ((0x80 as base.u32) >> neg),
- link: 0x0,
+ continued: 0,
length: 4)
args.src.skip32_fast!(actual: 4, worst_case: 4)
return ok
@@ -1623,7 +1618,7 @@
if args.src.available() <= 0 {
if whitespace_length > 0 {
args.dst.write_simple_token_fast!(
- value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
+ value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
whitespace_length = 0
}
if args.src.is_closed() {
@@ -1638,7 +1633,7 @@
if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE.
if whitespace_length > 0 {
args.dst.write_simple_token_fast!(
- value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
+ value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
whitespace_length = 0
}
return "#bad input"
@@ -1647,7 +1642,7 @@
args.src.skip32_fast!(actual: 1, worst_case: 1)
if (whitespace_length >= 0xFFFE) or (c == '\n') {
args.dst.write_simple_token_fast!(
- value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1)
+ value_major: 0, value_minor: 0, continued: 0, length: whitespace_length + 1)
whitespace_length = 0
if c == '\n' {
break.outer
diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c
index 8b82fb9..657e44a 100644
--- a/test/c/testlib/testlib.c
+++ b/test/c/testlib/testlib.c
@@ -1136,8 +1136,7 @@
uint16_t len = wuffs_base__token__length(t);
if (wuffs_base__token__value(t) != 0) {
- uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
- uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
+ uint16_t con = wuffs_base__token__continued(t) ? 1 : 0;
int32_t vmajor = wuffs_base__token__value_major(t);
if ((have.data.len - have.meta.wi) < 16) {
@@ -1149,8 +1148,7 @@
wuffs_base__store_u32be__no_bounds_check(ptr + 0x0, (uint32_t)(pos));
wuffs_base__store_u16be__no_bounds_check(ptr + 0x4, len);
- wuffs_base__store_u8__no_bounds_check(ptr + 0x0006, lp);
- wuffs_base__store_u8__no_bounds_check(ptr + 0x0007, ln);
+ wuffs_base__store_u16be__no_bounds_check(ptr + 0x6, con);
if (vmajor > 0) {
wuffs_base__store_u32be__no_bounds_check(ptr + 0x8, vmajor);
uint32_t vminor = wuffs_base__token__value_minor(t);
diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens
index 4274df8..1750ef5 100644
--- a/test/data/australian-abc-local-stations.tokens
+++ b/test/data/australian-abc-local-stations.tokens
Binary files differ
diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens
index d8feab9..d2518f2 100644
--- a/test/data/json-quirks.tokens
+++ b/test/data/json-quirks.tokens
Binary files differ
diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens
index 8fa0137..7bab270 100644
--- a/test/data/json-things.unformatted.tokens
+++ b/test/data/json-things.unformatted.tokens
Binary files differ
