From c1ae5d53fb0fa7ceb9d6fc7a60c87df958ce37fe Mon Sep 17 00:00:00 2001 From: Joel Klinghed Date: Sat, 27 Sep 2025 20:11:32 +0200 Subject: WIP --- test/java_tokens.cc | 551 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/java_uescape.cc | 9 + test/prefix_tree.cc | 26 +++ test/u.cc | 2 +- 4 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 test/java_tokens.cc (limited to 'test') diff --git a/test/java_tokens.cc b/test/java_tokens.cc new file mode 100644 index 0000000..df37409 --- /dev/null +++ b/test/java_tokens.cc @@ -0,0 +1,551 @@ +#include "java_tokens.hh" + +#include "errors.hh" +#include "io.hh" + +#include +#include +#include +#include + +using namespace std::literals::string_view_literals; + +namespace { + +class JavaTokens : public testing::TestWithParam { + protected: + static std::unique_ptr make_errors() { + return src::file_errors( + testing::UnitTest::GetInstance()->current_test_info()->name()); + } +}; + +} // namespace + +TEST_P(JavaTokens, empty_class) { + auto input = io::memory(R"(class Empty { +})"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kKeyword, ret->type); + EXPECT_EQ("class", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(0, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("Empty", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(6, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kSeparator, ret->type); + EXPECT_EQ("{", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(12, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kSeparator, ret->type); + EXPECT_EQ("}", ret->str); + EXPECT_EQ(2, ret->loc.line); + EXPECT_EQ(0, ret->loc.column); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, traditional_comment) { + auto input = io::memory(R"(/* this comment /* // /** ends here: */)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kComment, ret->type); + EXPECT_EQ("this comment /* // /** ends here:", ret->str); + EXPECT_EQ(1, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, single_line_comment) { + auto input = io::memory(R"(// this is a comment)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kComment, ret->type); + EXPECT_EQ("this is a comment", ret->str); + EXPECT_EQ(0, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, identifiers) { + auto input = io::memory(R"(String i3 αρετη MAX_VALUE isLetterOrDigit)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("String", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(0, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("i3", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(7, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("αρετη", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(10, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("MAX_VALUE", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(16, ret->loc.column); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kIdentifier, ret->type); + EXPECT_EQ("isLetterOrDigit", ret->str); + EXPECT_EQ(1, ret->loc.line); + EXPECT_EQ(26, ret->loc.column); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, int_literals) { + auto input = io::memory(R"( +0 2 0372 0xDada_Cafe 1996 0x00_FF__00_FF +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(0L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(2L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(250L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-623195394L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(1996L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(16711935L, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, long_literals) { + auto input = io::memory(R"( +0l 0777L 0x100000000L 2_147_483_648L 0xC0B0L +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(0, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(511, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(4294967296LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(2147483648LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(49328, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, int_literal_min_max) { + auto input = io::memory(R"( +2147483647 +-2147483648 +0x7fff_ffff +0177_7777_7777 +0b0111_1111_1111_1111_1111_1111_1111_1111 +0x8000_0000 +0200_0000_0000 +0b1000_0000_0000_0000_0000_0000_0000_0000 +0xffff_ffff +0377_7777_7777 +0b1111_1111_1111_1111_1111_1111_1111_1111 +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(2147483647L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kOperator, ret->type); + EXPECT_EQ("-", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-2147483647L - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(2147483647L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(2147483647L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(2147483647L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-2147483647L - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-2147483647L - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-2147483647L - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-1L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-1L, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type); + EXPECT_EQ(-1L, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, long_literal_min_max) { + auto input = io::memory(R"( +9223372036854775807L +-9223372036854775808L +0x7fff_ffff_ffff_ffffL +07_7777_7777_7777_7777_7777L +0b0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L +0x8000_0000_0000_0000L +010_0000_0000_0000_0000_0000L +0b1000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000L +0xffff_ffff_ffff_ffffL +017_7777_7777_7777_7777_7777L +0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(9223372036854775807LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kOperator, ret->type); + EXPECT_EQ("-", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(9223372036854775807LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(9223372036854775807LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(9223372036854775807LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-1LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-1LL, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type); + EXPECT_EQ(-1LL, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, float_literals) { + auto input = io::memory(R"( +1e1f 2.f .3f 0f 3.14f 6.022137e+23f +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(1e1F, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(2.F, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(.3F, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(0.F, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(3.14F, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type); + EXPECT_EQ(6.022137e+23F, ret->float_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, double_literals) { + auto input = io::memory(R"( +1e1 2. .3 0.0 3.14 1e-9d 1e137 +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(1e1, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(2., ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(.3, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(0.0, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(3.14, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(1e-9, ret->float_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type); + EXPECT_EQ(1e137, ret->float_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, bool_literals) { + auto input = io::memory(R"( +true false +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type); + EXPECT_TRUE(ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type); + EXPECT_EQ(0, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, char_literals) { + auto input = io::memory(R"( +'a' + +'%' + +'\t' + +'\\' + +'\'' + +'\u03a9' + +'\uFFFF' + +'\177' + +'™' +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ('a', ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ('%', ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ('\t', ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ('\\', ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ('\'', ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ(0x3a9, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ(0xffff, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ(0177, ret->int_value); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type); + EXPECT_EQ(0x2122, ret->int_value); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, string_literals) { + auto input = io::memory(R"( +"" +"\"" +"This is a string" +"This is a " + + "two-line string" +)"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ(R"(")", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("This is a string", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("This is a ", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kOperator, ret->type); + EXPECT_EQ("+", ret->str); + ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("two-line string", ret->str); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, string_escapes) { + auto input = io::memory(R"("\b\t\n\f\r\"\'\\\0\1\177")"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralString, ret->type); + EXPECT_EQ("\b\t\n\f\r\"'\\\0\1\177"sv, ret->str); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +TEST_P(JavaTokens, null) { + auto input = io::memory("null"); + auto tokens = java::open(std::move(input), make_errors(), + java::TokensConfig{.version = GetParam()}); + auto ret = tokens->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(java::Token::Type::kLiteralNull, ret->type); + ret = tokens->read(); + ASSERT_FALSE(ret.has_value()); + EXPECT_EQ(io::ReadError::Eof, ret.error()); +} + +INSTANTIATE_TEST_SUITE_P(AllVersions, JavaTokens, + testing::Values(java::Version::kJava8)); diff --git a/test/java_uescape.cc b/test/java_uescape.cc index a6657d8..a9f6746 100644 --- a/test/java_uescape.cc +++ b/test/java_uescape.cc @@ -132,6 +132,15 @@ TEST(java_uescape_u8, bad_surrogate) { EXPECT_EQ(io::ReadError::InvalidData, ret.error()); } +TEST(java_uescape_u8, issue) { + auto reader = u8::line::open(u8::java::open(io::memory(R"('\u03a9')"))); + auto ret = reader->read(); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ("'Ω'", ret.value()); + ret = reader->read(); + EXPECT_FALSE(ret.has_value()); +} + TEST(java_uescape_u16, escaped_escape) { auto reader = u16::java::open(io::memory(R"(\\u2122=\u2122)")); std::u16string tmp; diff --git a/test/prefix_tree.cc b/test/prefix_tree.cc index 6c00adb..86c8990 100644 --- a/test/prefix_tree.cc +++ b/test/prefix_tree.cc @@ -1,5 +1,7 @@ #include "prefix_tree.hh" +#include "str.hh" + #include TEST(prefix_tree, empty) { @@ -45,3 +47,27 @@ TEST(prefix_tree, sanity) { ASSERT_TRUE(ret.has_value()); EXPECT_EQ(3, ret.value()); } + +TEST(prefix_tree, many_and_long) { + auto builder = prefix_tree::builder(); + for (auto str : str::split( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut " + "enim ad minim veniam, quis nostrud exercitation ullamco laboris " + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in " + "reprehenderit in voluptate velit esse cillum dolore eu fugiat " + "nulla pariatur. Excepteur sint occaecat cupidatat non proident, " + "sunt in culpa qui officia deserunt mollit anim id est laborum.")) { + builder->add(str); + } + auto tree = builder->build(); + ASSERT_TRUE(tree.has_value()); + auto ret = prefix_tree::lookup(tree.value(), ""); + EXPECT_FALSE(ret.has_value()); + ret = prefix_tree::lookup(tree.value(), "Lorem"); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(5, ret.value()); + ret = prefix_tree::lookup(tree.value(), "cillum"); + ASSERT_TRUE(ret.has_value()); + EXPECT_EQ(6, ret.value()); +} diff --git a/test/u.cc b/test/u.cc index d43109b..d94f74e 100644 --- a/test/u.cc +++ b/test/u.cc @@ -711,7 +711,7 @@ TEST_P(UnicodeVersionTest, lookup_gc) { EXPECT_EQ(u::lookup_gc(0x483, GetParam()), u::GeneralCategory::MARK_NONSPACING); EXPECT_EQ(u::lookup_gc(0x93b, GetParam()), - u::GeneralCategory::MARK_SPACING_COMBINDING); + u::GeneralCategory::MARK_SPACING_COMBINING); EXPECT_EQ(u::lookup_gc(0x20de, GetParam()), u::GeneralCategory::MARK_SPACING_ENCLOSING); -- cgit v1.3