Add support for Java21 string template

core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java

@@ -27,6 +27,7 @@
 import com.sun.tools.javac.parser.Tokens.TokenKind;
 import com.sun.tools.javac.parser.UnicodeReader;
 import com.sun.tools.javac.util.Context;
+import java.util.Comparator;
 import java.util.Set;
 
 /** A wrapper around javac's lexer. */
@@ -71,22 +72,40 @@ public String stringVal() {
     }
   }
 
-  /** Lex the input and return a list of {@link RawTok}s. */
-  public static ImmutableList<RawTok> getTokens(
-      String source, Context context, Set<TokenKind> stopTokens) {
+  private static ImmutableList<Token> readAllToken(String source, Context context) {
     if (source == null) {
       return ImmutableList.of();
     }
     ScannerFactory fac = ScannerFactory.instance(context);
     char[] buffer = (source + EOF_COMMENT).toCharArray();
     Scanner scanner =
         new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length));
+    ImmutableList.Builder<Token> tokens = ImmutableList.builder();
+    do {
+      scanner.nextToken();
+      tokens.add(scanner.token());
+    } while (scanner.token().kind != TokenKind.EOF);
+    if (Runtime.version().feature() < 21) {
+      return tokens.build();
+    } else {
+      return ImmutableList.sortedCopyOf(Comparator.comparingInt(t -> t.pos), tokens.build());
+    }
+  }
+
+  /** Lex the input and return a list of {@link RawTok}s. */
+  public static ImmutableList<RawTok> getTokens(
+      String source, Context context, Set<TokenKind> stopTokens) {
+    if (source == null) {
+      return ImmutableList.of();
+    }
+    ImmutableList<Token> javacTokens = readAllToken(source, context);
+
+    var javacTokenIt = javacTokens.iterator();
     ImmutableList.Builder<RawTok> tokens = ImmutableList.builder();
     int end = source.length();
     int last = 0;
     do {
-      scanner.nextToken();
-      Token t = scanner.token();
+      Token t = javacTokenIt.next();
       if (t.comments != null) {
         for (Comment c : Lists.reverse(t.comments)) {
           if (last < c.getSourcePos(0)) {
@@ -104,6 +123,8 @@ public static ImmutableList<RawTok> getTokens(
         break;
       }
       if (last < t.pos) {
+        /* If the current token is not immediately following the previous one,
+        treat the gap as a token */
         tokens.add(new RawTok(null, null, last, t.pos));
       }
       tokens.add(
@@ -113,7 +134,7 @@ public static ImmutableList<RawTok> getTokens(
               t.pos,
               t.endPos));
       last = t.endPos;
-    } while (scanner.token().kind != TokenKind.EOF);
+    } while (javacTokenIt.hasNext());
     if (last < end) {
       tokens.add(new RawTok(null, null, last, end));
     }

core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java

@@ -23,6 +23,8 @@
 import com.sun.source.tree.ExpressionTree;
 import com.sun.source.tree.PatternCaseLabelTree;
 import com.sun.source.tree.PatternTree;
+import com.sun.source.tree.StringTemplateTree;
+import java.util.Iterator;
 import javax.lang.model.element.Name;
 
 /**
@@ -86,4 +88,38 @@ protected void variableName(Name name) {
       visit(name);
     }
   }
+
+  @Override
+  public Void visitStringTemplate(StringTemplateTree node, Void unused) {
+
+    scan(node.getProcessor(), null);
+
+    token(".");
+
+    Iterator<? extends ExpressionTree> exprIt = node.getExpressions().iterator();
+
+    Iterator<String> fragIt = node.getFragments().iterator();
+
+    var firstFrag = fragIt.next();
+
+    token("\"" + firstFrag);
+    while (fragIt.hasNext()) {
+      if (exprIt.hasNext()) {
+        ExpressionTree expr = exprIt.next();
+        token("\\");
+        token("{");
+        scan(expr, null);
+        token("}");
+      }
+
+      String frag = fragIt.next();
+      if (!fragIt.hasNext()) {
+        token(frag + "\"");
+      } else {
+        token(frag);
+      }
+    }
+
+    return null;
+  }
 }

core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java

@@ -59,7 +59,8 @@ public class FormatterIntegrationTest {
               "SwitchDouble",
               "SwitchUnderscore",
               "I880",
-              "Unnamed")
+              "Unnamed",
+              "StringTemplate")
           .build();
 
   @Parameters(name = "{index}: {0}")

core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java

@@ -492,4 +492,24 @@ public void removeTrailingTabsInComments() throws Exception {
                 + "  }\n"
                 + "}\n");
   }
+
+  @Test
+  public void stringTemplateTests() throws Exception {
+    assertThat(
+            new Formatter()
+                .formatSource(
+                    "public class Foo {\n"
+                        + "    String test(){\n"
+                        + "        var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n"
+                        + "        var nested = STR.\"template \\{example.  foo()+ STR.\"templateInner\\{  example}\"}xxx }\";\n"
+                        + "    }\n"
+                        + "}\n"))
+        .isEqualTo(
+            "public class Foo {\n"
+                + "  String test() {\n"
+                + "    var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n"
+                + "    var nested = STR.\"template \\{example.foo() + STR.\"templateInner\\{example}\"}xxx }\";\n"
+                + "  }\n"
+                + "}\n");
+  }
 }

core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input

@@ -0,0 +1,10 @@
+public class StringTemplates {
+    void test(){
+        var m = STR."template \{example}xxx";
+        var nested = STR."template \{example.foo()+ STR."templateInner\{example}"}xxx }";
+        var nestNested = STR."template \{example0.
+                    foo() +
+                    STR."templateInner\{example1.test(STR."\{example2
+                        }")}"}xxx }";
+    }
+}

core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output

@@ -0,0 +1,9 @@
+public class StringTemplates {
+  void test() {
+    var m = STR."template \{example}xxx";
+    var nested = STR."template \{example.foo() + STR."templateInner\{example}"}xxx }";
+    var nestNested =
+        STR."template \{example0.foo()
+            + STR."templateInner\{example1.test(STR."\{example2}")}"}xxx }";
+  }
+}