1 files changed, 92 insertions, 0 deletions
diff --git a/src/java_tokens.hh b/src/java_tokens.hh
new file mode 100644
index 0000000..6fbefcb
--- /dev/null
+++ b/src/java_tokens.hh
@@ -0,0 +1,92 @@
+#ifndef JAVA_TOKENS_HH
+#define JAVA_TOKENS_HH
+
+#include "errors.hh"
+#include "io.hh"
+#include "java_version.hh"  // IWYU pragma: export
+#include "location.hh"
+
+#include <expected>
+#include <memory>
+#include <string_view>
+
+namespace java {
+
+using src::Location;
+
+struct Token {
+  enum class Type : uint8_t {
+    // str is content of comment, excluding heading or trailing slash and star.
+    // int_value is number of stars at head, 0 for single line, 1 for /* and
+    // 2 for /** and so on.
+    kComment,
+
+    // str is identifier
+    kIdentifier,
+
+    // str is keyword, int_value is Keyword index
+    kKeyword,
+
+    // str is separator, int_value is Separator index
+    kSeparator,
+
+    // str is operator, int_value is Operator index
+    kOperator,
+
+    // int_value is literal value
+    kLiteralInt,
+
+    // int_value is literal value
+    kLiteralLong,
+
+    // str is literal value
+    kLiteralString,
+
+    // int_value is literal value as unicode code-point
+    kLiteralCharacter,
+
+    kLiteralNull,
+
+    // float_value is literal value
+    kLiteralFloatingPoint,
+
+    // float_value is literal value
+    kLiteralDoubleFloatingPoint,
+
+    // int_value is literal value, 0 = false, anything else = true
+    kLiteralBoolean,
+
+    kError,
+  };
+
+  Type type;
+  Location loc;
+  std::string_view str;
+  int64_t int_value{0};
+  double float_value{0};
+};
+
+struct TokensConfig {
+  // Source version of Java file
+  Version version = Version::kMax;
+};
+
+class Tokens {
+ public:
+  virtual ~Tokens() = default;
+
+  virtual std::expected<Token, io::ReadError> read() = 0;
+
+ protected:
+  Tokens() = default;
+  Tokens(Tokens const&) = delete;
+  Tokens& operator=(Tokens const&) = delete;
+};
+
+[[nodiscard]] std::unique_ptr<Tokens> open(std::unique_ptr<io::Reader> reader,
+                                           std::unique_ptr<src::Errors>,
+                                           TokensConfig config = {});
+
+}  // namespace java
+
+#endif  // JAVA_TOKENS_HH