Allow underscores in numeric constants
diff --git a/go.mod b/go.mod
index d3f2fc8..bfed505 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/google/wuffs
-go 1.12
+go 1.13
require (
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
diff --git a/lang/token/token.go b/lang/token/token.go
index c1e6ced..b46e4dc 100644
--- a/lang/token/token.go
+++ b/lang/token/token.go
@@ -89,14 +89,18 @@
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || (c == '_') || ('0' <= c && c <= '9')
}
-func hexaNumeric(c byte) bool {
- return ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') || ('0' <= c && c <= '9')
+func hexaNumericUnderscore(c byte) bool {
+ return ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') || (c == '_') || ('0' <= c && c <= '9')
}
func numeric(c byte) bool {
return ('0' <= c && c <= '9')
}
+func numericUnderscore(c byte) bool {
+ return (c == '_') || ('0' <= c && c <= '9')
+}
+
func hasPrefix(a []byte, s string) bool {
if len(s) == 0 {
return true
@@ -113,6 +117,19 @@
return true
}
+// checkNumericUnderscores rejects consecutive or trailing underscores.
+func checkNumericUnderscores(a []byte) bool {
+ prevUnderscore := false
+ for _, c := range a {
+ currUnderscore := c == '_'
+ if prevUnderscore && currUnderscore {
+ return false
+ }
+ prevUnderscore = currUnderscore
+ }
+ return !prevUnderscore
+}
+
func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error) {
line := uint32(1)
loop:
@@ -185,12 +202,10 @@
if numeric(c) {
// TODO: 0b11 binary numbers.
- //
- // TODO: allow underscores like 0b1000_0000_1111?
- j, isDigit := i+1, numeric
+ j, isDigit := i+1, numericUnderscore
if c == '0' && j < len(src) {
if next := src[j]; next == 'x' || next == 'X' {
- j, isDigit = j+1, hexaNumeric
+ j, isDigit = j+1, hexaNumericUnderscore
} else if numeric(next) {
return nil, nil, fmt.Errorf("token: legacy octal syntax at %s:%d", filename, line)
}
@@ -200,6 +215,9 @@
return nil, nil, fmt.Errorf("token: constant too long at %s:%d", filename, line)
}
}
+ if !checkNumericUnderscores(src[i:j]) {
+ return nil, nil, fmt.Errorf("token: invalid numeric literal at %s:%d", filename, line)
+ }
id, err := m.Insert(string(src[i:j]))
if err != nil {
return nil, nil, err
