WIPWIP

4 files changed, 587 insertions, 1 deletions

diff --git a/test/java_tokens.cc b/test/java_tokens.cc
new file mode 100644
index 0000000..df37409
--- /dev/null
+++ b/test/java_tokens.cc
@@ -0,0 +1,551 @@
+#include "java_tokens.hh"
+
+#include "errors.hh"
+#include "io.hh"
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+using namespace std::literals::string_view_literals;
+
+namespace {
+
+class JavaTokens : public testing::TestWithParam<java::Version> {
+ protected:
+  static std::unique_ptr<src::Errors> make_errors() {
+    return src::file_errors(
+        testing::UnitTest::GetInstance()->current_test_info()->name());
+  }
+};
+
+}  // namespace
+
+TEST_P(JavaTokens, empty_class) {
+  auto input = io::memory(R"(class Empty {
+})");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kKeyword, ret->type);
+  EXPECT_EQ("class", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("Empty", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(6, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kSeparator, ret->type);
+  EXPECT_EQ("{", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(12, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kSeparator, ret->type);
+  EXPECT_EQ("}", ret->str);
+  EXPECT_EQ(2, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, traditional_comment) {
+  auto input = io::memory(R"(/* this comment /* // /** ends here: */)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ("this comment /* // /** ends here:", ret->str);
+  EXPECT_EQ(1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, single_line_comment) {
+  auto input = io::memory(R"(// this is a comment)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ("this is a comment", ret->str);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, identifiers) {
+  auto input = io::memory(R"(String i3 αρετη MAX_VALUE isLetterOrDigit)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("String", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("i3", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(7, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("αρετη", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(10, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("MAX_VALUE", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(16, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("isLetterOrDigit", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(26, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, int_literals) {
+  auto input = io::memory(R"(
+0    2    0372    0xDada_Cafe    1996    0x00_FF__00_FF
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(0L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(250L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-623195394L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(1996L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(16711935L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, long_literals) {
+  auto input = io::memory(R"(
+0l    0777L    0x100000000L    2_147_483_648L    0xC0B0L
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(511, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(4294967296LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(2147483648LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(49328, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, int_literal_min_max) {
+  auto input = io::memory(R"(
+2147483647
+-2147483648
+0x7fff_ffff
+0177_7777_7777
+0b0111_1111_1111_1111_1111_1111_1111_1111
+0x8000_0000
+0200_0000_0000
+0b1000_0000_0000_0000_0000_0000_0000_0000
+0xffff_ffff
+0377_7777_7777
+0b1111_1111_1111_1111_1111_1111_1111_1111
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("-", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, long_literal_min_max) {
+  auto input = io::memory(R"(
+9223372036854775807L
+-9223372036854775808L
+0x7fff_ffff_ffff_ffffL
+07_7777_7777_7777_7777_7777L
+0b0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L
+0x8000_0000_0000_0000L
+010_0000_0000_0000_0000_0000L
+0b1000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000L
+0xffff_ffff_ffff_ffffL
+017_7777_7777_7777_7777_7777L
+0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("-", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, float_literals) {
+  auto input = io::memory(R"(
+1e1f    2.f    .3f    0f    3.14f    6.022137e+23f
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(1e1F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(2.F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(.3F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(0.F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(3.14F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(6.022137e+23F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, double_literals) {
+  auto input = io::memory(R"(
+1e1    2.    .3    0.0    3.14    1e-9d    1e137
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e1, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(2., ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(.3, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(0.0, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(3.14, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e-9, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e137, ret->float_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, bool_literals) {
+  auto input = io::memory(R"(
+true false
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type);
+  EXPECT_TRUE(ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, char_literals) {
+  auto input = io::memory(R"(
+'a'
+
+'%'
+
+'\t'
+
+'\\'
+
+'\''
+
+'\u03a9'
+
+'\uFFFF'
+
+'\177'
+
+'™'
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('a', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('%', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\t', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\\', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\'', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0x3a9, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0xffff, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0177, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0x2122, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, string_literals) {
+  auto input = io::memory(R"(
+""
+"\""
+"This is a string"
+"This is a " +
+    "two-line string"
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ(R"(")", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("This is a string", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("This is a ", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("+", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("two-line string", ret->str);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, string_escapes) {
+  auto input = io::memory(R"("\b\t\n\f\r\"\'\\\0\1\177")");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("\b\t\n\f\r\"'\\\0\1\177"sv, ret->str);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, null) {
+  auto input = io::memory("null");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralNull, ret->type);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+INSTANTIATE_TEST_SUITE_P(AllVersions, JavaTokens,
+                         testing::Values(java::Version::kJava8));
diff --git a/test/java_uescape.cc b/test/java_uescape.cc
index a6657d8..a9f6746 100644
--- a/test/java_uescape.cc
+++ b/test/java_uescape.cc
@@ -132,6 +132,15 @@ TEST(java_uescape_u8, bad_surrogate) {
   EXPECT_EQ(io::ReadError::InvalidData, ret.error());
 }
 
+TEST(java_uescape_u8, issue) {
+  auto reader = u8::line::open(u8::java::open(io::memory(R"('\u03a9')")));
+  auto ret = reader->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ("'Ω'", ret.value());
+  ret = reader->read();
+  EXPECT_FALSE(ret.has_value());
+}
+
 TEST(java_uescape_u16, escaped_escape) {
   auto reader = u16::java::open(io::memory(R"(\\u2122=\u2122)"));
   std::u16string tmp;
diff --git a/test/prefix_tree.cc b/test/prefix_tree.cc
index 6c00adb..86c8990 100644
--- a/test/prefix_tree.cc
+++ b/test/prefix_tree.cc
@@ -1,5 +1,7 @@
 #include "prefix_tree.hh"
 
+#include "str.hh"
+
 #include <gtest/gtest.h>
 
 TEST(prefix_tree, empty) {
@@ -45,3 +47,27 @@ TEST(prefix_tree, sanity) {
   ASSERT_TRUE(ret.has_value());
   EXPECT_EQ(3, ret.value());
 }
+
+TEST(prefix_tree, many_and_long) {
+  auto builder = prefix_tree::builder();
+  for (auto str : str::split(
+           "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
+           "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut "
+           "enim ad minim veniam, quis nostrud exercitation ullamco laboris "
+           "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in "
+           "reprehenderit in voluptate velit esse cillum dolore eu fugiat "
+           "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
+           "sunt in culpa qui officia deserunt mollit anim id est laborum.")) {
+    builder->add(str);
+  }
+  auto tree = builder->build();
+  ASSERT_TRUE(tree.has_value());
+  auto ret = prefix_tree::lookup(tree.value(), "");
+  EXPECT_FALSE(ret.has_value());
+  ret = prefix_tree::lookup(tree.value(), "Lorem");
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(5, ret.value());
+  ret = prefix_tree::lookup(tree.value(), "cillum");
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(6, ret.value());
+}
diff --git a/test/u.cc b/test/u.cc
index d43109b..d94f74e 100644
--- a/test/u.cc
+++ b/test/u.cc
@@ -711,7 +711,7 @@ TEST_P(UnicodeVersionTest, lookup_gc) {
   EXPECT_EQ(u::lookup_gc(0x483, GetParam()),
             u::GeneralCategory::MARK_NONSPACING);
   EXPECT_EQ(u::lookup_gc(0x93b, GetParam()),
-            u::GeneralCategory::MARK_SPACING_COMBINDING);
+            u::GeneralCategory::MARK_SPACING_COMBINING);
   EXPECT_EQ(u::lookup_gc(0x20de, GetParam()),
             u::GeneralCategory::MARK_SPACING_ENCLOSING);