java: Add tokens support for Java 21

Some new keywords, I opted to modify java-8 grammar to use the new
names, even if they are not going to match anything. Makes the
tokenizer easier to write.

2 files changed, 435 insertions, 0 deletions

diff --git a/data/java-21/tokens.grammar b/data/java-21/tokens.grammar
new file mode 100644
index 0000000..db935b2
--- /dev/null
+++ b/data/java-21/tokens.grammar
@@ -0,0 +1,423 @@
+InputElement:
+  WhiteSpace
+  Comment
+  Token
+
+Token:
+  Identifier
+  Keyword
+  Literal
+  Separator
+  Operator
+
+Comment:
+  TraditionalComment
+  EndOfLineComment
+
+TraditionalComment:
+  / * CommentTail
+
+CommentTail:
+  * CommentTailStar
+  NotStar CommentTail
+
+CommentTailStar:
+  /
+  * CommentTailStar
+  NotStarNotSlash CommentTail
+
+NotStar:
+  InputCharacter but not *
+  LineTerminator
+
+NotStarNotSlash:
+  InputCharacter but not * or /
+  LineTerminator
+
+EndOfLineComment:
+  / / {InputCharacter}
+
+Identifier:
+  IdentifierChars but not a ReservedKeyword or BooleanLiteral or NullLiteral
+
+IdentifierChars:
+  JavaLetter {JavaLetterOrDigit}
+
+Keyword:
+  ReservedKeyword
+  ContextualKeyword
+
+ReservedKeyword:
+  abstract
+  assert
+  boolean
+  break
+  byte
+  case
+  catch
+  char
+  class
+  const
+  continue
+  default
+  do
+  double
+  else
+  enum
+  extends
+  final
+  finally
+  float
+  for
+  goto
+  if
+  implements
+  import
+  instanceof
+  int
+  interface
+  long
+  native
+  new
+  package
+  private
+  protected
+  public
+  return
+  short
+  static
+  strictfp
+  super
+  switch
+  synchronized
+  this
+  throw
+  throws
+  transient
+  try
+  void
+  volatile
+  while
+  _
+
+ContextualKeyword:
+  exports
+  module
+  non-sealed
+  open
+  opens
+  permits
+  provides
+  record
+  requires
+  sealed
+  to
+  transitive
+  uses
+  var
+  when
+  with
+  yield
+
+Literal:
+  IntegerLiteral
+  FloatingPointLiteral
+  BooleanLiteral
+  CharacterLiteral
+  StringLiteral
+  TextBlock
+  NullLiteral
+
+IntegerLiteral:
+  DecimalIntegerLiteral
+  HexIntegerLiteral
+  OctalIntegerLiteral
+  BinaryIntegerLiteral
+
+DecimalIntegerLiteral:
+  DecimalNumeral [IntegerTypeSuffix]
+
+HexIntegerLiteral:
+  HexNumeral [IntegerTypeSuffix]
+
+OctalIntegerLiteral:
+  OctalNumeral [IntegerTypeSuffix]
+
+BinaryIntegerLiteral:
+  BinaryNumeral [IntegerTypeSuffix]
+
+IntegerTypeSuffix:
+  l
+  L
+
+DecimalNumeral:
+  0
+  NonZeroDigit [Digits]
+  NonZeroDigit Underscores Digits
+
+NonZeroDigit:
+  1
+  2
+  3
+  4
+  5
+  6
+  7
+  8
+  9
+
+Digits:
+  Digit
+  Digit [DigitsAndUnderscores] Digit
+
+Digit:
+  0
+  NonZeroDigit
+
+DigitsAndUnderscores:
+  DigitOrUnderscore {DigitOrUnderscore}
+
+DigitOrUnderscore:
+  Digit
+  _
+
+Underscores:
+  _ {_}
+
+HexNumeral:
+  0 x HexDigits
+  0 X HexDigits
+
+HexDigits:
+  HexDigit
+  HexDigit [HexDigitsAndUnderscores] HexDigit
+
+HexDigit:
+  0
+  1
+  2
+  3
+  4
+  5
+  6
+  7
+  8
+  9
+  a
+  b
+  c
+  d
+  e
+  f
+  A
+  B
+  C
+  D
+  E
+  F
+
+HexDigitsAndUnderscores:
+  HexDigitOrUnderscore {HexDigitOrUnderscore}
+
+HexDigitOrUnderscore:
+  HexDigit
+  _
+
+OctalNumeral:
+  0 OctalDigits
+  0 Underscores OctalDigits
+
+OctalDigits:
+  OctalDigit
+  OctalDigit [OctalDigitsAndUnderscores] OctalDigit
+
+OctalDigit:
+  0
+  1
+  2
+  3
+  4
+  5
+  6
+  7
+
+OctalDigitsAndUnderscores:
+  OctalDigitOrUnderscore {OctalDigitOrUnderscore}
+
+OctalDigitOrUnderscore:
+  OctalDigit
+  _
+
+BinaryNumeral:
+  0 b BinaryDigits
+  0 B BinaryDigits
+
+BinaryDigits:
+  BinaryDigit
+  BinaryDigit [BinaryDigitsAndUnderscores] BinaryDigit
+
+BinaryDigit:
+  0
+  1
+
+BinaryDigitsAndUnderscores:
+  BinaryDigitOrUnderscore {BinaryDigitOrUnderscore}
+
+BinaryDigitOrUnderscore:
+  BinaryDigit
+  _
+
+FloatingPointLiteral:
+  DecimalFloatingPointLiteral
+  HexadecimalFloatingPointLiteral
+
+DecimalFloatingPointLiteral:
+  Digits . [Digits] [ExponentPart] [FloatTypeSuffix]
+  . Digits [ExponentPart] [FloatTypeSuffix]
+  Digits ExponentPart [FloatTypeSuffix]
+  Digits [ExponentPart] FloatTypeSuffix
+
+ExponentPart:
+  ExponentIndicator SignedInteger
+
+ExponentIndicator:
+  e
+  E
+
+SignedInteger:
+  [Sign] Digits
+
+Sign:
+  +
+  -
+
+FloatTypeSuffix:
+  f
+  F
+  d
+  D
+
+HexadecimalFloatingPointLiteral:
+  HexSignificand BinaryExponent [FloatTypeSuffix]
+
+HexSignificand:
+  HexNumeral [.]
+  0 x [HexDigits] . HexDigits
+  0 X [HexDigits] . HexDigits
+
+BinaryExponent:
+  BinaryExponentIndicator SignedInteger
+
+BinaryExponentIndicator:
+  p
+  P
+
+BooleanLiteral:
+  true
+  false
+
+CharacterLiteral:
+  ' SingleCharacter '
+  ' EscapeSequence '
+
+SingleCharacter:
+  InputCharacter but not ' or \
+
+StringLiteral:
+  " {StringCharacter} "
+
+StringCharacter:
+  InputCharacter but not " or \
+  EscapeSequence
+
+TextBlock:
+  " " " {TextBlockWhiteSpace} LineTerminator {TextBlockCharacter} " " "
+
+TextBlockWhiteSpace:
+  WhiteSpace but not LineTerminator
+
+TextBlockCharacter:
+  InputCharacter but not \
+  EscapeSequence
+  LineTerminator
+
+EscapeSequence:
+  \ b
+  \ s
+  \ t
+  \ n
+  \ f
+  \ r
+  \ LineTerminator
+  \ "
+  \ '
+  \ \
+  OctalEscape
+
+OctalEscape:
+  \ OctalDigit
+  \ OctalDigit OctalDigit
+  \ ZeroToThree OctalDigit OctalDigit
+
+ZeroToThree:
+  0
+  1
+  2
+  3
+
+NullLiteral:
+  null
+
+Separator:
+  (
+  )
+  {
+  }
+  [
+  ]
+  ;
+  ,
+  .
+  ...
+  @
+  ::
+
+Operator:
+  =
+  >
+  <
+  !
+  ~
+  ?
+  :
+  ->
+  ==
+  >=
+  <=
+  !=
+  &&
+  ||
+  ++
+  --
+  +
+  -
+  *
+  /
+  &
+  |
+  ^
+  %
+  <<
+  >>
+  >>>
+  +=
+  -=
+  *=
+  /=
+  &=
+  |=
+  ^=
+  %=
+  <<=
+  >>=
+  >>>=
diff --git a/data/java-8/tokens.grammar b/data/java-8/tokens.grammar
index 3521ac0..3941b94 100644
--- a/data/java-8/tokens.grammar
+++ b/data/java-8/tokens.grammar
@@ -43,7 +43,13 @@ Identifier:
 IdentifierChars:
   JavaLetter {JavaLetterOrDigit}
 
+# Java 8 only has reserved keywords, but use modern names
+# here to make a shared tokenizer simpler.
 Keyword:
+  ReservedKeyword
+  ContextualKeyword
+
+ReservedKeyword:
   abstract
   continue
   for
@@ -95,14 +101,20 @@ Keyword:
   super
   while
 
+ContextualKeyword:
+
 Literal:
   IntegerLiteral
   FloatingPointLiteral
   BooleanLiteral
   CharacterLiteral
   StringLiteral
+  TextBlock
   NullLiteral
 
+# Java 8 doesn't have TextBlock, but add it as newer grammers have it
+TextBlock:
+
 IntegerLiteral:
   DecimalIntegerLiteral
   HexIntegerLiteral