java: Add tokens

Only parses Java 8 tokens for now.

1 files changed, 80 insertions, 0 deletions

diff --git a/src/grammar.hh b/src/grammar.hh
new file mode 100644
index 0000000..13beec4
--- /dev/null
+++ b/src/grammar.hh
@@ -0,0 +1,80 @@
+#ifndef GRAMMAR_HH
+#define GRAMMAR_HH
+
+#include "errors.hh"
+#include "io.hh"
+
+#include <cstdint>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace grammar {
+
+struct Element;
+
+struct Symbol {
+  enum class Type : uint8_t {
+    // value == terminal, as UTF-8
+    kTerminal,
+
+    // element != nullptr
+    kNonTerminal,
+
+    // char_class != 0
+    kCharacterClass,
+  };
+
+  Type type;
+
+  enum class Optional : uint8_t {
+    // Symbol is NOT optional.
+    kRequired,
+    // Symbol is optional.
+    kZeroOrOne,
+    // Symbol is optional and can repeat.
+    kZeroOrMore,
+    // Symbol should be excluded from previous symbol.
+    // Example: InputCharacter but not * or /
+    kExcluded,
+  };
+
+  Optional optional{Optional::kRequired};
+
+  uint8_t char_class{0};
+
+  Element const* element{nullptr};
+  std::string value;
+};
+
+struct Definition {
+  std::vector<Symbol> symbols;
+};
+
+struct Element {
+  std::string name;
+
+  std::vector<Definition> definitions;
+};
+
+class Grammar {
+ public:
+  virtual ~Grammar() = default;
+
+  [[nodiscard]]
+  virtual Element const& root() const = 0;
+
+ protected:
+  Grammar() = default;
+
+  Grammar(Grammar const&) = delete;
+  Grammar& operator=(Grammar const&) = delete;
+};
+
+std::unique_ptr<Grammar> load(std::unique_ptr<io::Reader> reader,
+                              std::vector<std::string> const& character_classes,
+                              src::Errors& errors);
+
+}  // namespace grammar
+
+#endif  // GRAMMAR_HH