java: Add tokens

Only parses Java 8 tokens for now.

1 files changed, 606 insertions, 0 deletions

diff --git a/test/java_tokens.cc b/test/java_tokens.cc
new file mode 100644
index 0000000..1c69196
--- /dev/null
+++ b/test/java_tokens.cc
@@ -0,0 +1,606 @@
+#include "java_tokens.hh"
+
+#include "errors.hh"
+#include "io.hh"
+
+#include <gtest/gtest.h>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+using namespace std::literals::string_view_literals;
+
+namespace {
+
+class JavaTokens : public testing::TestWithParam<java::Version> {
+ protected:
+  static std::unique_ptr<src::Errors> make_errors() {
+    return src::file_errors(
+        testing::UnitTest::GetInstance()->current_test_info()->name());
+  }
+};
+
+}  // namespace
+
+TEST_P(JavaTokens, empty_class) {
+  auto input = io::memory(R"(class Empty {
+})");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kKeyword, ret->type);
+  EXPECT_EQ("class", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("Empty", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(6, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kSeparator, ret->type);
+  EXPECT_EQ("{", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(12, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kSeparator, ret->type);
+  EXPECT_EQ("}", ret->str);
+  EXPECT_EQ(2, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, traditional_comment) {
+  auto input = io::memory(R"(/* this comment /* // /** ends here: */)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ("this comment /* // /** ends here:", ret->str);
+  EXPECT_EQ(1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, traditional_multiline_comment) {
+  auto input = io::memory(R"(/* this comment
+ /* //
+/** ends here: */)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ(R"(this comment
+ /* //
+/** ends here:)",
+            ret->str);
+  EXPECT_EQ(1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, javadoc_comment) {
+  auto input = io::memory(R"(/** Returns something complicated */)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ("Returns something complicated", ret->str);
+  EXPECT_EQ(2, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, javadoc_multiline_comment) {
+  auto input = io::memory(R"(
+/**
+ * Returns something complicated.
+ * @param foo something
+ * @param bar something else
+ */)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ(R"(* Returns something complicated.
+ * @param foo something
+ * @param bar something else)",
+            ret->str);
+  EXPECT_EQ(2, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, single_line_comment) {
+  auto input = io::memory(R"(// this is a comment)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kComment, ret->type);
+  EXPECT_EQ("this is a comment", ret->str);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, identifiers) {
+  auto input = io::memory(R"(String i3 αρετη MAX_VALUE isLetterOrDigit)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("String", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(0, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("i3", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(7, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("αρετη", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(10, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("MAX_VALUE", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(16, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kIdentifier, ret->type);
+  EXPECT_EQ("isLetterOrDigit", ret->str);
+  EXPECT_EQ(1, ret->loc.line);
+  EXPECT_EQ(26, ret->loc.column);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, int_literals) {
+  auto input = io::memory(R"(
+0    2    0372    0xDada_Cafe    1996    0x00_FF__00_FF
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(0L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(250L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-623195394L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(1996L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(16711935L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, long_literals) {
+  auto input = io::memory(R"(
+0l    0777L    0x100000000L    2_147_483_648L    0xC0B0L
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(511, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(4294967296LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(2147483648LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(49328, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, int_literal_min_max) {
+  auto input = io::memory(R"(
+2147483647
+-2147483648
+0x7fff_ffff
+0177_7777_7777
+0b0111_1111_1111_1111_1111_1111_1111_1111
+0x8000_0000
+0200_0000_0000
+0b1000_0000_0000_0000_0000_0000_0000_0000
+0xffff_ffff
+0377_7777_7777
+0b1111_1111_1111_1111_1111_1111_1111_1111
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("-", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(2147483647L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-2147483647L - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralInt, ret->type);
+  EXPECT_EQ(-1L, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, long_literal_min_max) {
+  auto input = io::memory(R"(
+9223372036854775807L
+-9223372036854775808L
+0x7fff_ffff_ffff_ffffL
+07_7777_7777_7777_7777_7777L
+0b0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L
+0x8000_0000_0000_0000L
+010_0000_0000_0000_0000_0000L
+0b1000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000_0000L
+0xffff_ffff_ffff_ffffL
+017_7777_7777_7777_7777_7777L
+0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111L
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("-", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(9223372036854775807LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-9223372036854775807LL - 1, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralLong, ret->type);
+  EXPECT_EQ(-1LL, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, float_literals) {
+  auto input = io::memory(R"(
+1e1f    2.f    .3f    0f    3.14f    6.022137e+23f
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(1e1F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(2.F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(.3F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(0.F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(3.14F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralFloatingPoint, ret->type);
+  EXPECT_EQ(6.022137e+23F, ret->float_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, double_literals) {
+  auto input = io::memory(R"(
+1e1    2.    .3    0.0    3.14    1e-9d    1e137
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e1, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(2., ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(.3, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(0.0, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(3.14, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e-9, ret->float_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralDoubleFloatingPoint, ret->type);
+  EXPECT_EQ(1e137, ret->float_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, bool_literals) {
+  auto input = io::memory(R"(
+true false
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type);
+  EXPECT_TRUE(ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralBoolean, ret->type);
+  EXPECT_EQ(0, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, char_literals) {
+  auto input = io::memory(R"(
+'a'
+
+'%'
+
+'\t'
+
+'\\'
+
+'\''
+
+'\u03a9'
+
+'\uFFFF'
+
+'\177'
+
+'™'
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('a', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('%', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\t', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\\', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ('\'', ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0x3a9, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0xffff, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0177, ret->int_value);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralCharacter, ret->type);
+  EXPECT_EQ(0x2122, ret->int_value);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, string_literals) {
+  auto input = io::memory(R"(
+""
+"\""
+"This is a string"
+"This is a " +
+    "two-line string"
+)");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ(R"(")", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("This is a string", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("This is a ", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kOperator, ret->type);
+  EXPECT_EQ("+", ret->str);
+  ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("two-line string", ret->str);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, string_escapes) {
+  auto input = io::memory(R"("\b\t\n\f\r\"\'\\\0\1\177")");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralString, ret->type);
+  EXPECT_EQ("\b\t\n\f\r\"'\\\0\1\177"sv, ret->str);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+TEST_P(JavaTokens, null) {
+  auto input = io::memory("null");
+  auto tokens = java::open(std::move(input), make_errors(),
+                           java::TokensConfig{.version = GetParam()});
+  auto ret = tokens->read();
+  ASSERT_TRUE(ret.has_value());
+  EXPECT_EQ(java::Token::Type::kLiteralNull, ret->type);
+  ret = tokens->read();
+  ASSERT_FALSE(ret.has_value());
+  EXPECT_EQ(io::ReadError::Eof, ret.error());
+}
+
+INSTANTIATE_TEST_SUITE_P(AllVersions, JavaTokens,
+                         testing::Values(java::Version::kJava8));