Skip to content

Commit 577fda1

Browse files
chandlercchandlerc
authored
Speed up type literal lexing and make it more strict. (#4430)
This rejects type literals with more digits than we can lex without APInt's help, and using a custom diagnostic. This is a pretty arbitrary implementation limit, I'm wide open to even more strict rules here. Despite no special casing and a very simplistic approach, by not using APInt this completely eliminates the lexing overhead for `i32` in the generated compilation benchmark where that specific type literal is very common. We see a 10% improvement in lexing there: ``` BM_CompileAPIFileDenseDecls<Phase::Lex>/256 39.0µs ± 4% 34.8µs ± 2% -10.86% (p=0.000 n=19+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/1024 180µs ± 1% 158µs ± 2% -12.22% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/4096 731µs ± 2% 641µs ± 1% -12.31% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/16384 3.20ms ± 2% 2.86ms ± 2% -10.47% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Lex>/65536 13.8ms ± 1% 12.4ms ± 2% -9.78% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Lex>/262144 64.0ms ± 2% 58.4ms ± 2% -8.70% (p=0.000 n=19+18) ``` This starts to fix a TODO in the diagnostic for these by giving a reasonably good diagnostic about a very large type literal. However, in practice it regresses the diagnostics because error tokens produce noisy extraneous diagnostics from parse and check currently. Leaving the TODO there, and I have a follow-up PR to start improving the extraneous diagnostics.
1 parent b67d031 commit 577fda1

File tree

6 files changed

+108
-135
lines changed

6 files changed

+108
-135
lines changed

‎toolchain/check/testdata/basics/type_literals.carbon‎

Lines changed: 20 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,18 @@ var test_i15: i15;
3939
// CHECK:STDERR:
4040
var test_i1000000000: i1000000000;
4141
// TODO: This diagnostic is not very good.
42-
// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+4]]:33: error: integer literal with value 10000000000000000000 does not fit in i32 [IntLiteralTooLargeForI32]
42+
// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+12]]:33: error: expected expression [ExpectedExpr]
4343
// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000;
4444
// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~
4545
// CHECK:STDERR:
46+
// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+8]]:33: error: semantics TODO: `HandleInvalidParse` [SemanticsTodo]
47+
// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000;
48+
// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~
49+
// CHECK:STDERR:
50+
// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+4]]:34: error: found a type literal with a bit width using 20 digits, which is greater than the limit of 18 [TooManyTypeBitWidthDigits]
51+
// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000;
52+
// CHECK:STDERR: ^
53+
// CHECK:STDERR:
4654
var test_i10000000000000000000: i10000000000000000000;
4755

4856
// --- uN.carbon
@@ -76,10 +84,18 @@ var test_u15: u15;
7684
// CHECK:STDERR:
7785
var test_u1000000000: u1000000000;
7886
// TODO: This diagnostic is not very good.
79-
// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+4]]:33: error: integer literal with value 10000000000000000000 does not fit in i32 [IntLiteralTooLargeForI32]
87+
// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+12]]:33: error: expected expression [ExpectedExpr]
88+
// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000;
89+
// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~
90+
// CHECK:STDERR:
91+
// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+8]]:33: error: semantics TODO: `HandleInvalidParse` [SemanticsTodo]
8092
// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000;
8193
// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~
8294
// CHECK:STDERR:
95+
// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+4]]:34: error: found a type literal with a bit width using 20 digits, which is greater than the limit of 18 [TooManyTypeBitWidthDigits]
96+
// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000;
97+
// CHECK:STDERR: ^
98+
// CHECK:STDERR:
8399
var test_u10000000000000000000: u10000000000000000000;
84100

85101
// --- fail_fN_bad_width.carbon
@@ -183,58 +199,7 @@ var test_f128: f128;
183199
// CHECK:STDOUT: %.6: i32 = int_literal 1000000000 [template]
184200
// CHECK:STDOUT: }
185201
// CHECK:STDOUT:
186-
// CHECK:STDOUT: imports {
187-
// CHECK:STDOUT: %Core: <namespace> = namespace file.%Core.import, [template] {
188-
// CHECK:STDOUT: .Int = %import_ref
189-
// CHECK:STDOUT: import Core//prelude
190-
// CHECK:STDOUT: import Core//prelude/operators
191-
// CHECK:STDOUT: import Core//prelude/types
192-
// CHECK:STDOUT: import Core//prelude/operators/arithmetic
193-
// CHECK:STDOUT: import Core//prelude/operators/as
194-
// CHECK:STDOUT: import Core//prelude/operators/bitwise
195-
// CHECK:STDOUT: import Core//prelude/operators/comparison
196-
// CHECK:STDOUT: import Core//prelude/types/bool
197-
// CHECK:STDOUT: }
198-
// CHECK:STDOUT: %import_ref: %Int.type = import_ref Core//prelude/types, inst+29, loaded [template = constants.%Int]
199-
// CHECK:STDOUT: }
200-
// CHECK:STDOUT:
201-
// CHECK:STDOUT: file {
202-
// CHECK:STDOUT: package: <namespace> = namespace [template] {
203-
// CHECK:STDOUT: .Core = imports.%Core
204-
// CHECK:STDOUT: .test_i0 = %test_i0
205-
// CHECK:STDOUT: .test_i1 = %test_i1
206-
// CHECK:STDOUT: .test_i15 = %test_i15
207-
// CHECK:STDOUT: .test_i1000000000 = %test_i1000000000
208-
// CHECK:STDOUT: .test_i10000000000000000000 = %test_i10000000000000000000
209-
// CHECK:STDOUT: }
210-
// CHECK:STDOUT: %Core.import = import Core
211-
// CHECK:STDOUT: %i0.ref: <error> = name_ref i0, <error> [template = <error>]
212-
// CHECK:STDOUT: %test_i0.var: ref <error> = var test_i0
213-
// CHECK:STDOUT: %test_i0: ref <error> = bind_name test_i0, %test_i0.var
214-
// CHECK:STDOUT: %.loc12_14.1: i32 = int_literal 1 [template = constants.%.1]
215-
// CHECK:STDOUT: %int.make_type_signed.loc12: init type = call constants.%Int(%.loc12_14.1) [template = constants.%.3]
216-
// CHECK:STDOUT: %.loc12_14.2: type = value_of_initializer %int.make_type_signed.loc12 [template = constants.%.3]
217-
// CHECK:STDOUT: %.loc12_14.3: type = converted %int.make_type_signed.loc12, %.loc12_14.2 [template = constants.%.3]
218-
// CHECK:STDOUT: %test_i1.var: ref %.3 = var test_i1
219-
// CHECK:STDOUT: %test_i1: ref %.3 = bind_name test_i1, %test_i1.var
220-
// CHECK:STDOUT: %.loc17_15.1: i32 = int_literal 15 [template = constants.%.4]
221-
// CHECK:STDOUT: %int.make_type_signed.loc17: init type = call constants.%Int(%.loc17_15.1) [template = constants.%.5]
222-
// CHECK:STDOUT: %.loc17_15.2: type = value_of_initializer %int.make_type_signed.loc17 [template = constants.%.5]
223-
// CHECK:STDOUT: %.loc17_15.3: type = converted %int.make_type_signed.loc17, %.loc17_15.2 [template = constants.%.5]
224-
// CHECK:STDOUT: %test_i15.var: ref %.5 = var test_i15
225-
// CHECK:STDOUT: %test_i15: ref %.5 = bind_name test_i15, %test_i15.var
226-
// CHECK:STDOUT: %.loc22_23.1: i32 = int_literal 1000000000 [template = constants.%.6]
227-
// CHECK:STDOUT: %int.make_type_signed.loc22: init type = call constants.%Int(%.loc22_23.1) [template = <error>]
228-
// CHECK:STDOUT: %.loc22_23.2: type = value_of_initializer %int.make_type_signed.loc22 [template = <error>]
229-
// CHECK:STDOUT: %.loc22_23.3: type = converted %int.make_type_signed.loc22, %.loc22_23.2 [template = <error>]
230-
// CHECK:STDOUT: %test_i1000000000.var: ref <error> = var test_i1000000000
231-
// CHECK:STDOUT: %test_i1000000000: ref <error> = bind_name test_i1000000000, %test_i1000000000.var
232-
// CHECK:STDOUT: %int.make_type_signed.loc28: init type = call constants.%Int(<invalid>) [template = <error>]
233-
// CHECK:STDOUT: %.loc28_33.1: type = value_of_initializer %int.make_type_signed.loc28 [template = <error>]
234-
// CHECK:STDOUT: %.loc28_33.2: type = converted %int.make_type_signed.loc28, %.loc28_33.1 [template = <error>]
235-
// CHECK:STDOUT: %test_i10000000000000000000.var: ref <error> = var test_i10000000000000000000
236-
// CHECK:STDOUT: %test_i10000000000000000000: ref <error> = bind_name test_i10000000000000000000, %test_i10000000000000000000.var
237-
// CHECK:STDOUT: }
202+
// CHECK:STDOUT: file {}
238203
// CHECK:STDOUT:
239204
// CHECK:STDOUT: fn @Int(%size.param_patt: i32) -> type = "int.make_type_signed";
240205
// CHECK:STDOUT:
@@ -310,58 +275,7 @@ var test_f128: f128;
310275
// CHECK:STDOUT: %.6: i32 = int_literal 1000000000 [template]
311276
// CHECK:STDOUT: }
312277
// CHECK:STDOUT:
313-
// CHECK:STDOUT: imports {
314-
// CHECK:STDOUT: %Core: <namespace> = namespace file.%Core.import, [template] {
315-
// CHECK:STDOUT: .UInt = %import_ref
316-
// CHECK:STDOUT: import Core//prelude
317-
// CHECK:STDOUT: import Core//prelude/operators
318-
// CHECK:STDOUT: import Core//prelude/types
319-
// CHECK:STDOUT: import Core//prelude/operators/arithmetic
320-
// CHECK:STDOUT: import Core//prelude/operators/as
321-
// CHECK:STDOUT: import Core//prelude/operators/bitwise
322-
// CHECK:STDOUT: import Core//prelude/operators/comparison
323-
// CHECK:STDOUT: import Core//prelude/types/bool
324-
// CHECK:STDOUT: }
325-
// CHECK:STDOUT: %import_ref: %UInt.type = import_ref Core//prelude/types, inst+43, loaded [template = constants.%UInt]
326-
// CHECK:STDOUT: }
327-
// CHECK:STDOUT:
328-
// CHECK:STDOUT: file {
329-
// CHECK:STDOUT: package: <namespace> = namespace [template] {
330-
// CHECK:STDOUT: .Core = imports.%Core
331-
// CHECK:STDOUT: .test_u0 = %test_u0
332-
// CHECK:STDOUT: .test_u1 = %test_u1
333-
// CHECK:STDOUT: .test_u15 = %test_u15
334-
// CHECK:STDOUT: .test_u1000000000 = %test_u1000000000
335-
// CHECK:STDOUT: .test_u10000000000000000000 = %test_u10000000000000000000
336-
// CHECK:STDOUT: }
337-
// CHECK:STDOUT: %Core.import = import Core
338-
// CHECK:STDOUT: %u0.ref: <error> = name_ref u0, <error> [template = <error>]
339-
// CHECK:STDOUT: %test_u0.var: ref <error> = var test_u0
340-
// CHECK:STDOUT: %test_u0: ref <error> = bind_name test_u0, %test_u0.var
341-
// CHECK:STDOUT: %.loc12_14.1: i32 = int_literal 1 [template = constants.%.1]
342-
// CHECK:STDOUT: %int.make_type_unsigned.loc12: init type = call constants.%UInt(%.loc12_14.1) [template = constants.%.3]
343-
// CHECK:STDOUT: %.loc12_14.2: type = value_of_initializer %int.make_type_unsigned.loc12 [template = constants.%.3]
344-
// CHECK:STDOUT: %.loc12_14.3: type = converted %int.make_type_unsigned.loc12, %.loc12_14.2 [template = constants.%.3]
345-
// CHECK:STDOUT: %test_u1.var: ref %.3 = var test_u1
346-
// CHECK:STDOUT: %test_u1: ref %.3 = bind_name test_u1, %test_u1.var
347-
// CHECK:STDOUT: %.loc17_15.1: i32 = int_literal 15 [template = constants.%.4]
348-
// CHECK:STDOUT: %int.make_type_unsigned.loc17: init type = call constants.%UInt(%.loc17_15.1) [template = constants.%.5]
349-
// CHECK:STDOUT: %.loc17_15.2: type = value_of_initializer %int.make_type_unsigned.loc17 [template = constants.%.5]
350-
// CHECK:STDOUT: %.loc17_15.3: type = converted %int.make_type_unsigned.loc17, %.loc17_15.2 [template = constants.%.5]
351-
// CHECK:STDOUT: %test_u15.var: ref %.5 = var test_u15
352-
// CHECK:STDOUT: %test_u15: ref %.5 = bind_name test_u15, %test_u15.var
353-
// CHECK:STDOUT: %.loc22_23.1: i32 = int_literal 1000000000 [template = constants.%.6]
354-
// CHECK:STDOUT: %int.make_type_unsigned.loc22: init type = call constants.%UInt(%.loc22_23.1) [template = <error>]
355-
// CHECK:STDOUT: %.loc22_23.2: type = value_of_initializer %int.make_type_unsigned.loc22 [template = <error>]
356-
// CHECK:STDOUT: %.loc22_23.3: type = converted %int.make_type_unsigned.loc22, %.loc22_23.2 [template = <error>]
357-
// CHECK:STDOUT: %test_u1000000000.var: ref <error> = var test_u1000000000
358-
// CHECK:STDOUT: %test_u1000000000: ref <error> = bind_name test_u1000000000, %test_u1000000000.var
359-
// CHECK:STDOUT: %int.make_type_unsigned.loc28: init type = call constants.%UInt(<invalid>) [template = <error>]
360-
// CHECK:STDOUT: %.loc28_33.1: type = value_of_initializer %int.make_type_unsigned.loc28 [template = <error>]
361-
// CHECK:STDOUT: %.loc28_33.2: type = converted %int.make_type_unsigned.loc28, %.loc28_33.1 [template = <error>]
362-
// CHECK:STDOUT: %test_u10000000000000000000.var: ref <error> = var test_u10000000000000000000
363-
// CHECK:STDOUT: %test_u10000000000000000000: ref <error> = bind_name test_u10000000000000000000, %test_u10000000000000000000.var
364-
// CHECK:STDOUT: }
278+
// CHECK:STDOUT: file {}
365279
// CHECK:STDOUT:
366280
// CHECK:STDOUT: fn @UInt(%size.param_patt: i32) -> type = "int.make_type_unsigned";
367281
// CHECK:STDOUT:

‎toolchain/diagnostics/diagnostic_kind.def‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ CARBON_DIAGNOSTIC_KIND(MultiLineStringWithDoubleQuotes)
4242
CARBON_DIAGNOSTIC_KIND(NoWhitespaceAfterCommentIntroducer)
4343
CARBON_DIAGNOSTIC_KIND(TooManyDigits)
4444
CARBON_DIAGNOSTIC_KIND(TooManyTokens)
45+
CARBON_DIAGNOSTIC_KIND(TooManyTypeBitWidthDigits)
4546
CARBON_DIAGNOSTIC_KIND(TrailingComment)
4647
CARBON_DIAGNOSTIC_KIND(UnicodeEscapeMissingBracedDigits)
4748
CARBON_DIAGNOSTIC_KIND(UnicodeEscapeSurrogate)

‎toolchain/lex/lex.cpp‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1166,10 +1166,6 @@ auto Lexer::LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset)
11661166
// Too short to form one of these tokens.
11671167
return LexResult::NoMatch();
11681168
}
1169-
if (word[1] < '1' || word[1] > '9') {
1170-
// Doesn't start with a valid initial digit.
1171-
return LexResult::NoMatch();
1172-
}
11731169

11741170
TokenKind kind;
11751171
switch (word.front()) {
@@ -1186,17 +1182,49 @@ auto Lexer::LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset)
11861182
return LexResult::NoMatch();
11871183
};
11881184

1185+
// No leading zeros allowed.
1186+
if ('1' > word[1] || word[1] > '9') {
1187+
return LexResult::NoMatch();
1188+
}
1189+
11891190
llvm::StringRef suffix = word.substr(1);
1190-
if (!CanLexInt(emitter_, suffix)) {
1191+
1192+
// Type bit-widths can't usefully be large integers so we restrict to small
1193+
// ones that are especially easy to parse into a normal integer variable by
1194+
// restricting the number of digits to round trip.
1195+
int64_t suffix_value;
1196+
constexpr ssize_t DigitLimit =
1197+
std::numeric_limits<decltype(suffix_value)>::digits10;
1198+
if (suffix.size() > DigitLimit) {
1199+
// See if this is not actually a type literal.
1200+
if (!llvm::all_of(suffix, IsDecimalDigit)) {
1201+
return LexResult::NoMatch();
1202+
}
1203+
1204+
// Otherwise, diagnose and produce an error token.
1205+
CARBON_DIAGNOSTIC(TooManyTypeBitWidthDigits, Error,
1206+
"found a type literal with a bit width using {0} digits, "
1207+
"which is greater than the limit of {1}",
1208+
size_t, size_t);
1209+
emitter_.Emit(word.begin() + 1, TooManyTypeBitWidthDigits, suffix.size(),
1210+
DigitLimit);
11911211
return LexTokenWithPayload(TokenKind::Error, word.size(), byte_offset);
11921212
}
1193-
llvm::APInt suffix_value;
1194-
if (suffix.getAsInteger(10, suffix_value)) {
1195-
return LexResult::NoMatch();
1213+
1214+
// It's tempting to do something more clever because we know the length ahead
1215+
// of time, but we expect these to be short (1-3 digits) and profiling doesn't
1216+
// show the loop as hot in the short cases.
1217+
suffix_value = suffix[0] - '0';
1218+
for (char c : suffix.drop_front()) {
1219+
if (!IsDecimalDigit(c)) {
1220+
return LexResult::NoMatch();
1221+
}
1222+
suffix_value = suffix_value * 10 + (c - '0');
11961223
}
11971224

11981225
return LexTokenWithPayload(
1199-
kind, buffer_.value_stores_->ints().Add(std::move(suffix_value)).index,
1226+
kind,
1227+
buffer_.value_stores_->ints().Add(llvm::APInt(64, suffix_value)).index,
12001228
byte_offset);
12011229
}
12021230

‎toolchain/lex/tokenized_buffer_test.cpp‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <gmock/gmock.h>
88
#include <gtest/gtest.h>
99

10+
#include <cmath>
1011
#include <forward_list>
1112
#include <iterator>
1213

@@ -1023,27 +1024,54 @@ TEST_F(LexerTest, TypeLiterals) {
10231024
}
10241025

10251026
TEST_F(LexerTest, TypeLiteralTooManyDigits) {
1027+
// We increase the number of digits until the first one that is to large.
1028+
Testing::MockDiagnosticConsumer consumer;
1029+
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
1030+
DiagnosticKind::TooManyTypeBitWidthDigits,
1031+
DiagnosticLevel::Error, 1, 2, _)));
10261032
std::string code = "i";
1033+
// A 128-bit APInt should be plenty large, but if needed in the future it can
1034+
// be widened without issue.
1035+
llvm::APInt bits = llvm::APInt::getZero(128);
1036+
for ([[maybe_unused]] int _ : llvm::seq(1, 30)) {
1037+
code.append("9");
1038+
bits = bits * 10 + 9;
1039+
auto [buffer, value_stores] =
1040+
compile_helper_.GetTokenizedBufferWithSharedValueStore(code, &consumer);
1041+
if (buffer.has_errors()) {
1042+
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
1043+
{.kind = TokenKind::FileStart},
1044+
{.kind = TokenKind::Error, .text = code},
1045+
{.kind = TokenKind::FileEnd},
1046+
}));
1047+
break;
1048+
}
1049+
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
1050+
{.kind = TokenKind::FileStart},
1051+
{.kind = TokenKind::IntTypeLiteral, .text = code},
1052+
{.kind = TokenKind::FileEnd},
1053+
}));
1054+
auto token = buffer.tokens().begin()[1];
1055+
EXPECT_TRUE(llvm::APInt::isSameValue(
1056+
value_stores.ints().Get(buffer.GetTypeLiteralSize(token)), bits));
1057+
}
1058+
1059+
// Make sure we can also gracefully reject very large number of digits without
1060+
// crashing or hanging, and show the correct number.
10271061
constexpr int Count = 10000;
1062+
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
1063+
DiagnosticKind::TooManyTypeBitWidthDigits,
1064+
DiagnosticLevel::Error, 1, 2,
1065+
HasSubstr(llvm::formatv(" {0} ", Count)))));
1066+
code = "i";
10281067
code.append(Count, '9');
1029-
1030-
Testing::MockDiagnosticConsumer consumer;
1031-
EXPECT_CALL(consumer,
1032-
HandleDiagnostic(IsSingleDiagnostic(
1033-
DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2,
1034-
HasSubstr(llvm::formatv(" {0} ", Count)))));
10351068
auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer);
1036-
EXPECT_TRUE(buffer.has_errors());
1037-
ASSERT_THAT(buffer,
1038-
HasTokens(llvm::ArrayRef<ExpectedToken>{
1039-
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
1040-
{.kind = TokenKind::Error,
1041-
.line = 1,
1042-
.column = 1,
1043-
.indent_column = 1,
1044-
.text = code},
1045-
{.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2},
1046-
}));
1069+
ASSERT_TRUE(buffer.has_errors());
1070+
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
1071+
{.kind = TokenKind::FileStart},
1072+
{.kind = TokenKind::Error, .text = code},
1073+
{.kind = TokenKind::FileEnd},
1074+
}));
10471075
}
10481076

10491077
TEST_F(LexerTest, DiagnosticTrailingComment) {

‎toolchain/testing/compile_helper.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ auto CompileHelper::GetTokenizedBuffer(llvm::StringRef text,
1717
return token_storage_.front();
1818
}
1919

20-
auto CompileHelper::GetTokenizedBufferWithSharedValueStore(llvm::StringRef text)
20+
auto CompileHelper::GetTokenizedBufferWithSharedValueStore(
21+
llvm::StringRef text, DiagnosticConsumer* consumer)
2122
-> std::pair<Lex::TokenizedBuffer&, SharedValueStores&> {
22-
auto& tokens = GetTokenizedBuffer(text);
23+
auto& tokens = GetTokenizedBuffer(text, consumer);
2324
return {tokens, value_store_storage_.front()};
2425
}
2526

‎toolchain/testing/compile_helper.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ class CompileHelper {
2525
-> Lex::TokenizedBuffer&;
2626

2727
// Returns the result of lex along with shared values.
28-
auto GetTokenizedBufferWithSharedValueStore(llvm::StringRef text)
28+
auto GetTokenizedBufferWithSharedValueStore(
29+
llvm::StringRef text, DiagnosticConsumer* consumer = nullptr)
2930
-> std::pair<Lex::TokenizedBuffer&, SharedValueStores&>;
3031

3132
// Returns the result of parse.

0 commit comments

Comments
 (0)