1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
#ifndef JAVA_TOKENS_HH
#define JAVA_TOKENS_HH
#include "errors.hh"
#include "io.hh"
#include "java_version.hh" // IWYU pragma: export
#include "location.hh"
#include <expected>
#include <memory>
#include <string_view>
namespace java {
using src::Location;
struct Token {
enum class Type : uint8_t {
// str is content of comment, excluding heading or trailing slash and star.
// int_value is number of stars at head, 0 for single line, 1 for /* and
// 2 for /** and so on.
kComment,
// str is identifier
kIdentifier,
// str is keyword, int_value is Keyword index
kReservedKeyword,
// str is keyword, int_value is Keyword index
kContextualKeyword,
// str is separator, int_value is Separator index
kSeparator,
// str is operator, int_value is Operator index
kOperator,
// int_value is literal value
kLiteralInt,
// int_value is literal value
kLiteralLong,
// str is literal value
kLiteralString,
// int_value is literal value as unicode code-point
kLiteralCharacter,
kLiteralNull,
// float_value is literal value
kLiteralFloatingPoint,
// float_value is literal value
kLiteralDoubleFloatingPoint,
// int_value is literal value, 0 = false, anything else = true
kLiteralBoolean,
kError,
};
Type type;
Location loc;
std::string_view str;
int64_t int_value{0};
double float_value{0};
};
struct TokensConfig {
// Source version of Java file
Version version = Version::kMax;
};
class Tokens {
public:
virtual ~Tokens() = default;
virtual std::expected<Token, io::ReadError> read() = 0;
protected:
Tokens() = default;
Tokens(Tokens const&) = delete;
Tokens& operator=(Tokens const&) = delete;
};
[[nodiscard]] std::unique_ptr<Tokens> open(std::unique_ptr<io::Reader> reader,
std::unique_ptr<src::Errors>,
TokensConfig config = {});
} // namespace java
#endif // JAVA_TOKENS_HH
|
