Enrich the lexer to support the entire CEL lexicon

experimental/incremental/queries/ir.go

@@ -89,10 +89,11 @@ func (i IR) Execute(t *incremental.Task) (ir.File, error) {
 		iterx.Count(file.Imports())+1)
 	errors := make([]error, len(queries))
 	for j, decl := range iterx.Enumerate(file.Imports()) {
-		path, ok := decl.ImportPath().AsLiteral().AsString()
+		lit := decl.ImportPath().AsLiteral().AsString()
+		path := lit.Text()
 		path = ir.CanonicalizeFilePath(path)
 
-		if !ok {
+		if lit.IsZero() {
 			// The import path is already legalized in [parser.legalizeImport()], if it is not
 			// a valid path, we just set a [incremental.ZeroQuery] so that we don't get a nil
 			// query for index j.

experimental/internal/astx/encode.go

@@ -384,12 +384,12 @@ func (c *protoEncoder) expr(expr ast.ExprAny) *compilerpb.Expr {
 		proto := &compilerpb.Expr_Literal{
 			Span: c.span(expr),
 		}
-		if v, ok := expr.Token.AsInt(); ok {
+		if v, exact := expr.Token.AsNumber().AsInt(); exact {
 			proto.Value = &compilerpb.Expr_Literal_IntValue{IntValue: v}
-		} else if v, ok := expr.Token.AsFloat(); ok {
+		} else if v, exact := expr.Token.AsNumber().AsFloat(); exact {
 			proto.Value = &compilerpb.Expr_Literal_FloatValue{FloatValue: v}
-		} else if v, ok := expr.Token.AsString(); ok {
-			proto.Value = &compilerpb.Expr_Literal_StringValue{StringValue: v}
+		} else if v := expr.Token.AsString(); !v.IsZero() {
+			proto.Value = &compilerpb.Expr_Literal_StringValue{StringValue: v.Text()}
 		} else {
 			panic(fmt.Sprintf("protocompile/ast: ExprLiteral contains neither string nor int: %v", expr.Token))
 		}

experimental/internal/taxa/classify.go

@@ -27,7 +27,7 @@ import (
 
 // IsFloat checks whether or not tok is intended to be a floating-point literal.
 func IsFloat(tok token.Token) bool {
-	return IsFloatText(tok.Text())
+	return tok.AsNumber().HasFloatSyntax()
 }
 
 // IsFloatText checks whether or not the given number text is intended to be

experimental/internal/tokenmeta/meta.go

@@ -0,0 +1,53 @@
+// Copyright 2020-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package meta defines internal metadata types shared between the token package
+// and the lexer.
+package tokenmeta
+
+import "math/big"
+
+// Meta is a type defined in this package.
+type Meta interface{ meta() }
+
+type Number struct {
+	// The "used" value is whichever one of these is
+	// non-zero. If they're all zero, this is a zero-valued number.
+	Word  uint64
+	Float float64
+	Big   *big.Int
+
+	// Length of a prefix or suffix on this integer.
+	// The prefix is the base prefix; the suffix is any identifier
+	// characters that follow the last digit.
+	Prefix, Suffix uint32
+	FloatSyntax    bool
+	ThousandsSep   bool
+
+	SyntaxError bool // Whether parsing a concrete value failed.
+}
+
+type String struct {
+	// Post-processed string contents.
+	Text string
+
+	// Lengths of the sigil and quotes for this string
+	Prefix, Quote uint32
+
+	// Whether escaping or concatenation took place.
+	Escaped, Concatenated bool
+}
+
+func (Number) meta() {}
+func (String) meta() {}

experimental/ir/lower_eval.go

@@ -381,8 +381,8 @@ again:
 	case ast.ExprKindLiteral:
 		lit := key.AsLiteral()
 		if lit.Kind() == token.Number {
-			n, ok := lit.AsInt()
-			if ok && n < math.MaxInt32 {
+			n, exact := lit.AsNumber().AsInt()
+			if exact && n < math.MaxInt32 {
 				member = ty.MemberByNumber(int32(n))
 				if !member.IsZero() {
 					isNumber = true
@@ -393,7 +393,7 @@ again:
 			return Member{}
 		}
 
-		path, _ = lit.AsString()
+		path = lit.AsString().Text()
 		isString = true
 
 	default:
@@ -737,68 +737,69 @@ func (e *evaluator) evalLiteral(args evalArgs, expr ast.ExprLiteral, neg ast.Exp
 
 	switch expr.Kind() {
 	case token.Number:
+		lit := expr.AsNumber()
 		// Handle floats first, since all number formats can be used as floats.
 		if scalar.IsFloat() {
-			if n, ok := expr.AsFloat(); ok {
-				// If the number contains no decimal point, check that it has no
-				// 0x prefix. Hex literals are not permitted for float-typed
-				// values, but we don't know that until here, much later than
-				// all the other base checks in the compiler.
-				text := expr.Text()
-				if !taxa.IsFloatText(text) && (strings.HasPrefix(text, "0x") || strings.HasPrefix(text, "0X")) {
-					e.Errorf("unsupported base for %s", taxa.Float).Apply(
-						report.SuggestEdits(expr, "use a decimal literal instead", report.Edit{
-							Start: 0, End: len(text),
-							Replace: strconv.FormatFloat(n, 'g', 40, 64),
-						}),
-						report.Notef("Protobuf does not support hexadecimal %s", taxa.Float),
-					)
-				}
+			n, _ := lit.AsFloat()
+
+			// If the number contains no decimal point, check that it has no
+			// 0x prefix. Hex literals are not permitted for float-typed
+			// values, but we don't know that until here, much later than
+			// all the other base checks in the compiler.
+			text := expr.Text()
+			if !taxa.IsFloatText(text) && (strings.HasPrefix(text, "0x") || strings.HasPrefix(text, "0X")) {
+				e.Errorf("unsupported base for %s", taxa.Float).Apply(
+					report.SuggestEdits(expr, "use a decimal literal instead", report.Edit{
+						Start: 0, End: len(text),
+						Replace: strconv.FormatFloat(n, 'g', 40, 64),
+					}),
+					report.Notef("Protobuf does not support hexadecimal %s", taxa.Float),
+				)
+			}
 
-				if !neg.IsZero() {
-					n = -n
-				}
-				if scalar == predeclared.Float32 {
-					// This will, among other things, snap n to Infinity or zero
-					// if it is in-range for float64 but not float32.
-					n = float64(float32(n))
-				}
+			if !neg.IsZero() {
+				n = -n
+			}
+			if scalar == predeclared.Float32 {
+				// This will, among other things, snap n to Infinity or zero
+				// if it is in-range for float64 but not float32.
+				n = float64(float32(n))
+			}
 
-				// Emit a diagnostic if the value is snapped to infinity.
-				// TODO: Should we emit a diagnostic when rounding produces
-				// the value 0.0 but expr.Text() contains non-zero digits?
-				if math.IsInf(n, 0) {
-					d := e.Warnf("%s rounds to infinity", taxa.Float).Apply(
-						report.Snippetf(expr, "this value is beyond the dynamic range of `%s`", scalar),
-						report.SuggestEdits(expr, "replace with `inf`", report.Edit{
-							Start: 0, End: len(text),
-							Replace: "inf", // The sign is not part of the expression.
-						}),
-					)
-
-					// If possible, show the power-of-10 exponent of the value.
-					f := new(big.Float)
-					if _, _, err := f.Parse(expr.Text(), 0); err == nil {
-						maxExp := 308
-						if scalar == predeclared.Float32 {
-							maxExp = 38
-						}
-
-						exp2 := f.MantExp(nil)                      // ~ log2 f
-						exp10 := int(float64(exp2) / math.Log2(10)) // log10 f = log2 f / log2 10
-						d.Apply(report.Notef(
-							"this value is of order 1e%d; `%s` can only represent around 1e%d",
-							exp10, scalar, maxExp))
+			// Emit a diagnostic if the value is snapped to infinity.
+			// TODO: Should we emit a diagnostic when rounding produces
+			// the value 0.0 but expr.Text() contains non-zero digits?
+			if math.IsInf(n, 0) {
+				d := e.Warnf("%s rounds to infinity", taxa.Float).Apply(
+					report.Snippetf(expr, "this value is beyond the dynamic range of `%s`", scalar),
+					report.SuggestEdits(expr, "replace with `inf`", report.Edit{
+						Start: 0, End: len(text),
+						Replace: "inf", // The sign is not part of the expression.
+					}),
+				)
+
+				// If possible, show the power-of-10 exponent of the value.
+				f := new(big.Float)
+				if _, _, err := f.Parse(expr.Text(), 0); err == nil {
+					maxExp := 308
+					if scalar == predeclared.Float32 {
+						maxExp = 38
 					}
-				}
 
-				// 32-bit floats are stored as 64-bit floats; this conversion is
-				// lossless.
-				return rawValueBits(math.Float64bits(n)), true
+					exp2 := f.MantExp(nil)                      // ~ log2 f
+					exp10 := int(float64(exp2) / math.Log2(10)) // log10 f = log2 f / log2 10
+					d.Apply(report.Notef(
+						"this value is of order 1e%d; `%s` can only represent around 1e%d",
+						exp10, scalar, maxExp))
+				}
 			}
+
+			// 32-bit floats are stored as 64-bit floats; this conversion is
+			// lossless.
+			return rawValueBits(math.Float64bits(n)), true
 		}
 
-		if n, ok := expr.AsInt(); ok {
+		if n, exact := lit.AsInt(); exact && !lit.HasFloatSyntax() {
 			switch args.memberNumber {
 			case enumNumber:
 				return e.checkIntBounds(args, true, enumNumberBits, !neg.IsZero(), n)
@@ -816,7 +817,7 @@ func (e *evaluator) evalLiteral(args evalArgs, expr ast.ExprLiteral, neg ast.Exp
 			return e.checkIntBounds(args, scalar.IsSigned(), scalar.Bits(), !neg.IsZero(), n)
 		}
 
-		if n := expr.AsBigInt(); n != nil {
+		if n, exact := lit.AsBig(); exact && !lit.HasFloatSyntax() {
 			switch args.memberNumber {
 			case enumNumber:
 				return e.checkIntBounds(args, true, enumNumberBits, !neg.IsZero(), n)
@@ -833,10 +834,8 @@ func (e *evaluator) evalLiteral(args evalArgs, expr ast.ExprLiteral, neg ast.Exp
 			return e.checkIntBounds(args, scalar.IsSigned(), scalar.Bits(), !neg.IsZero(), n)
 		}
 
-		if _, ok := expr.AsFloat(); ok {
-			e.Error(args.mismatch(taxa.Float))
-			return 0, false
-		}
+		e.Error(args.mismatch(taxa.Float))
+		return 0, false
 
 	case token.String:
 		if scalar != predeclared.String && scalar != predeclared.Bytes {
@@ -853,7 +852,7 @@ func (e *evaluator) evalLiteral(args evalArgs, expr ast.ExprLiteral, neg ast.Exp
 			})
 		}
 
-		data, _ := expr.AsString()
+		data := expr.AsString().Text()
 		return newScalarBits(e.Context, data), true
 	}
 

experimental/ir/lower_imports.go

@@ -54,11 +54,11 @@ func buildImports(f File, r *report.Report, importer Importer) {
 	dedup := make(intern.Map[ast.DeclImport], iterx.Count(f.AST().Imports()))
 
 	for i, imp := range iterx.Enumerate(f.AST().Imports()) {
-		path, ok := imp.ImportPath().AsLiteral().AsString()
-		if !ok {
+		lit := imp.ImportPath().AsLiteral().AsString()
+		if lit.IsZero() {
 			continue // Already legalized in parser.legalizeImport()
 		}
-		path = canonicalizeImportPath(path, r, imp)
+		path := canonicalizeImportPath(lit.Text(), r, imp)
 		if path == "" {
 			continue
 		}
@@ -143,20 +143,19 @@ func buildImports(f File, r *report.Report, importer Importer) {
 // diagnoseCycle generates a diagnostic for an import cycle, showing each
 // import contributing to the cycle in turn.
 func diagnoseCycle(r *report.Report, cycle *ErrCycle) {
-	path, _ := cycle.Cycle[0].ImportPath().AsLiteral().AsString()
+	path := cycle.Cycle[0].ImportPath().AsLiteral().AsString().Text()
 	err := r.Errorf("detected cyclic import while importing %q", path)
 
 	for i, imp := range cycle.Cycle {
 		var message string
-		path, ok := imp.ImportPath().AsLiteral().AsString()
-		if ok {
+		if path := imp.ImportPath().AsLiteral().AsString(); !path.IsZero() {
 			switch i {
 			case 0:
 				message = "imported here"
 			case len(cycle.Cycle) - 1:
-				message = fmt.Sprintf("...which imports %q, completing the cycle", path)
+				message = fmt.Sprintf("...which imports %q, completing the cycle", path.Text())
 			default:
-				message = fmt.Sprintf("...which imports %q...", path)
+				message = fmt.Sprintf("...which imports %q...", path.Text())
 			}
 		}
 		err.Apply(

experimental/ir/lower_walk.go

@@ -52,13 +52,13 @@ func (w *walker) walk() {
 
 	c.syntax = syntax.Proto2
 	if syn := w.AST().Syntax(); !syn.IsZero() {
-		unquoted, ok := syn.Value().AsLiteral().AsString()
-		if !ok {
-			unquoted = syn.Value().Span().Text()
+		text := syn.Value().Span().Text()
+		if unquoted := syn.Value().AsLiteral().AsString(); !unquoted.IsZero() {
+			text = unquoted.Text()
 		}
 
 		// NOTE: This matches fallback behavior in parser/legalize_file.go.
-		c.syntax = syntax.Lookup(unquoted)
+		c.syntax = syntax.Lookup(text)
 		if c.syntax == syntax.Unknown {
 			if syn.IsEdition() {
 				// If they wrote edition = "garbage" they probably want *an*
@@ -193,7 +193,7 @@ func (w *walker) newType(def ast.DeclDef, parent any) Type {
 				if id := v.AsPath().AsIdent(); !id.IsZero() {
 					name = id.Text()
 				} else {
-					name, _ = v.AsLiteral().AsString()
+					name = v.AsLiteral().AsString().Text()
 				}
 
 				ty.raw.reservedNames = append(ty.raw.reservedNames, rawReservedName{

experimental/ir/testdata/tags/fields.proto.stderr.txt

@@ -1,3 +1,9 @@
+error: unexpected range expression in message field tag
+  --> testdata/tags/fields.proto:60:25
+   |
+60 |     optional int32 m3 = 1 to 2;
+   |                         ^^^^^^
+
 error: unexpected `:` after definition
   --> testdata/tags/fields.proto:61:26
    |
@@ -294,4 +300,4 @@ error: cannot find `a` in this scope
    |
    = help: the full name of this scope is `test.M`
 
-encountered 33 errors
+encountered 34 errors

experimental/ir/testdata/tags/values.proto.stderr.txt

@@ -1,3 +1,9 @@
+error: unexpected range expression in enum value
+  --> testdata/tags/values.proto:50:10
+   |
+50 |     M3 = 1 to 2;
+   |          ^^^^^^
+
 error: unexpected `:` after definition
   --> testdata/tags/values.proto:51:11
    |
@@ -228,4 +234,4 @@ error: mismatched types
    = note: expected: scalar type `int32`
               found: enum type `test.E`
 
-encountered 27 errors
+encountered 28 errors

experimental/parser/diagnostics_token.go

@@ -79,7 +79,7 @@ type errImpureString struct {
 
 // Diagnose implements [report.Diagnose].
 func (e errImpureString) Diagnose(d *report.Diagnostic) {
-	text, _ := e.lit.AsString()
+	text := e.lit.AsString().Text()
 	quote := e.lit.Text()[0]
 	d.Apply(
 		report.Message("non-canonical string literal %s", e.where.String()),

experimental/parser/legalize_decl.go

@@ -148,7 +148,8 @@ func legalizeRange(p *parser, parent classified, decl ast.DeclRange) {
 
 		case ast.ExprKindLiteral:
 			lit := expr.AsLiteral()
-			if name, ok := lit.AsString(); ok {
+			if str := lit.AsString(); !str.IsZero() {
+				name := str.Text()
 				if in == taxa.Extensions {
 					p.Error(errUnexpected{
 						what:  expr,
@@ -190,7 +191,7 @@ func legalizeRange(p *parser, parent classified, decl ast.DeclRange) {
 					break
 				}
 
-				if !lit.IsPureString() {
+				if !str.IsPure() {
 					p.Warn(errImpureString{lit.Token, in.In()})
 				}