From d196d51e07f50f3510c43ad375c5559b58860023 Mon Sep 17 00:00:00 2001 From: Joel Klinghed Date: Mon, 29 Sep 2025 09:39:49 +0200 Subject: java: Add tokens support for Java 21 Some new keywords, I opted to modify java-8 grammar to use the new names, even if they are not going to match anything. Makes the tokenizer easier to write. --- test/java_tokens.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/java_tokens.cc b/test/java_tokens.cc index 1c69196..cb1ae73 100644 --- a/test/java_tokens.cc +++ b/test/java_tokens.cc @@ -29,7 +29,7 @@ TEST_P(JavaTokens, empty_class) { java::TokensConfig{.version = GetParam()}); auto ret = tokens->read(); ASSERT_TRUE(ret.has_value()); - EXPECT_EQ(java::Token::Type::kKeyword, ret->type); + EXPECT_EQ(java::Token::Type::kReservedKeyword, ret->type); EXPECT_EQ("class", ret->str); EXPECT_EQ(1, ret->loc.line); EXPECT_EQ(0, ret->loc.column); @@ -602,5 +602,57 @@ TEST_P(JavaTokens, null) { EXPECT_EQ(io::ReadError::Eof, ret.error()); } +TEST_P(JavaTokens, textblock) { + auto input = io::memory(R"(String html = """ + + +

Hello, world

+ + + """;)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("String", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("html", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kOperator, ret->type); + EXPECT_EQ("=", ret->str); + ret = tokens->read(); + if (std::to_underlying(GetParam()) >= 15) { + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ(R"( + +

Hello, world

+ + +)", + ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kSeparator, ret->type); + EXPECT_EQ(";", ret->str); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); + } else { + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kError, ret->type); + } +} + INSTANTIATE_TEST_SUITE_P(AllVersions, JavaTokens, - testing::Values(java::Version::kJava8)); + testing::Values(java::Version::kJava8, + java::Version::kJava21)); -- cgit v1.3