Serialize inline source maps (base64-encoded "data url" stored in //# sourceMappingURL= comments)

PiperOrigin-RevId: 516248443

Author: rahul-kamat

src/com/google/javascript/jscomp/AbstractCompiler.java

@@ -528,7 +528,7 @@ public boolean hasColorAndSimplifiedJSDoc() {
    */
   public abstract void addInputSourceMap(String name, SourceMapInput sourceMap);
 
-  public abstract String getInputSourceMappingURL(String sourceFileName);
+  public abstract String getBase64SourceMapContents(String sourceFileName);
 
   abstract void addComments(String filename, List<Comment> comments);
 

src/com/google/javascript/jscomp/Compiler.java

@@ -20,6 +20,7 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.common.annotations.GwtIncompatible;
 import com.google.common.annotations.VisibleForTesting;
@@ -35,6 +36,7 @@
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
+import com.google.common.io.BaseEncoding;
 import com.google.debugging.sourcemap.SourceMapConsumerV3;
 import com.google.debugging.sourcemap.proto.Mapping.OriginalMapping;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
@@ -3302,26 +3304,20 @@ public void addInputSourceMap(String sourceFileName, SourceMapInput inputSourceM
   }
 
   /**
-   * Returns the <url> from a `//# sourceMappingURL=<url>` comment. This sourceMappingURL is a JS
-   * file's input source map, embedded into the input JS file with a `//# sourceMappingURL=<url>`
-   * comment.
-   *
-   * <p>We do not want TypedAST to support inline source maps (which are input source maps passed by
-   * embedding a `//# sourceMappingURL=<url>` where <url> is a base64-encoded "data url"). If the
-   * sourceMappingURL ends with a ".inline.map", return null.
+   * Returns the <encoded_source_map> from a `//#
+   * sourceMappingURL=data:application/json;base64,<encoded_source_map>` comment. This
+   * sourceMappingURL comment is a JS file's inline source map, embedded into the input JS file with
+   * a base64-encoded "data url" stored in `//# sourceMappingURL=` comment.
    */
   @Override
-  public @Nullable String getInputSourceMappingURL(String sourceFileName) {
+  public @Nullable String getBase64SourceMapContents(String sourceFileName) {
     SourceMapInput sourceMapInput = inputSourceMaps.getOrDefault(sourceFileName, null);
+    // This sourceMappingURL ends with a ".inline.map", and we want to get its base4-encoded
+    // contents (i.e. "data:application/json;base64,<encoded_source_map>").
     String sourceMappingURL = sourceMapInput != null ? sourceMapInput.getOriginalPath() : null;
-    if (sourceMappingURL != null && !sourceMappingURL.endsWith(".inline.map")) {
-      // Set sourceMappingURL as the name of the sourcemap file, not the original location/path of
-      // the sourcemap file on disk. E.g. turn "blaze-out/.../a.js.map" into "a.js.map"
-      sourceMappingURL =
-          sourceMappingURL.contains("/")
-              ? sourceMappingURL.substring(sourceMappingURL.lastIndexOf("/") + 1)
-              : sourceMappingURL;
-      return sourceMappingURL;
+    if (sourceMappingURL != null && sourceMappingURL.endsWith(".inline.map")) {
+      String unencoded = sourceMapInput.getRawSourceMapContents(); // This is the sourcemap.
+      return BaseEncoding.base64().encode(unencoded.getBytes(UTF_8));
     }
     return null;
   }
@@ -3925,6 +3921,8 @@ protected static class CompilerState implements Serializable {
     private final String idGeneratorMap;
     private final IdGenerator crossModuleIdGenerator;
     private final boolean runJ2clPasses;
+    // TODO(b/235404079): Stop serializing input source maps here since we serialize them in
+    // TypedAsts
     private final ImmutableMap<String, SourceMapInput> inputSourceMaps;
     private final ImmutableList<InputId> externs;
     private final ImmutableListMultimap<JSChunk, InputId> moduleToInputList;

src/com/google/javascript/jscomp/SourceMapInput.java

@@ -76,4 +76,12 @@ public SourceMapInput(SourceFile sourceFile) {
   public String getOriginalPath() {
     return sourceFile.getName();
   }
+
+  public @Nullable String getRawSourceMapContents() {
+    try {
+      return this.sourceFile.getCode();
+    } catch (IOException e) {
+      return null;
+    }
+  }
 }

src/com/google/javascript/jscomp/SourceMapResolver.java

@@ -103,5 +103,15 @@ private static boolean isAbsolute(String url) {
         .build();
   }
 
+  /**
+   * Returns the encoded source map with the base64 prefix attached. (i.e. prefix with
+   * "data:application/json;base64,")
+   *
+   * <p>E.g. turns "eyJ2ZXJzaW9uI..." into "data:application/json;base64,eyJ2ZXJzaW9uI..."
+   */
+  public static String addBase64PrefixToEncodedSourceMap(String encoded) {
+    return BASE64_URL_PREFIX + "application/json;" + BASE64_START + encoded;
+  }
+
   private SourceMapResolver() {}
 }

src/com/google/javascript/jscomp/serialization/TypedAstDeserializer.java

@@ -351,11 +351,15 @@ private static void addInputSourceMap(
       boolean resolveSourceMapAnnotations,
       boolean parseInlineSourceMaps) {
     SourceFile sourceFile = deserializer.getSourceFile();
-    String sourceMappingURL = deserializer.getSourceMappingURL();
+    String sourceMappingURL = deserializer.getSourceMappingURL(); // This is the encoded source map.
 
     if (sourceMappingURL != null && sourceMappingURL.length() > 0 && resolveSourceMapAnnotations) {
+      // base64EncodedSourceMap adds "data:application/json;base64," prefix to the sourceMappingURL
+      String base64EncodedSourceMap =
+          SourceMapResolver.addBase64PrefixToEncodedSourceMap(sourceMappingURL);
       SourceFile sourceMapSourceFile =
-          SourceMapResolver.extractSourceMap(sourceFile, sourceMappingURL, parseInlineSourceMaps);
+          SourceMapResolver.extractSourceMap(
+              sourceFile, base64EncodedSourceMap, parseInlineSourceMaps);
       if (sourceMapSourceFile != null) {
         compiler.addInputSourceMap(sourceFile.getName(), new SourceMapInput(sourceMapSourceFile));
       }

src/com/google/javascript/jscomp/serialization/TypedAstSerializer.java

@@ -118,13 +118,17 @@ private LazyAst serializeScriptNode(Node script) {
     AstNode scriptProto = visit(script);
     this.subtreeSourceFiles.clear();
 
-    String sourceMappingURL = compiler.getInputSourceMappingURL(script.getSourceFileName());
+    String encodedSourceMap = compiler.getBase64SourceMapContents(script.getSourceFileName());
 
     LazyAst.Builder lazyAstBuilder =
         LazyAst.newBuilder().setScript(scriptProto.toByteString()).setSourceFile(sourceFile);
 
-    if (sourceMappingURL != null) {
-      lazyAstBuilder.setSourceMappingUrl(sourceMappingURL);
+    if (encodedSourceMap != null) {
+      // This is the encoded source map taken from the inline sourcemap comment. It does not include
+      // the base64 prefix.
+      // E.g. We serialize "eyJ2ZXJzaW9uI..." from the "//# sourceMappingURL=
+      // data:application/json;base64,eyJ2ZXJzaW9uI..." comment.
+      lazyAstBuilder.setSourceMappingUrl(encodedSourceMap);
     }
 
     return lazyAstBuilder.build();

src/com/google/javascript/rhino/typed_ast/typed_ast.proto

@@ -34,8 +34,12 @@ message LazyAst {
   bytes script = 1;
   // 1-based index into TypedAst::source_file_pool
   uint32 source_file = 2;
-  // The URL for a source map associated with this source file (i.e. the URL
-  // specified in "//# sourceMappingURL=" comments).
+  // The encoded source map taken from the inline sourcemap comment
+  // (base64-encoded "data url" stored in `//# sourceMappingURL=` comment).
+  // E.g. If the last line of a .closure.js file is "//# sourceMappingURL=
+  // data:application/json;base64,eyJ2ZXJzaW9uI...", then we will serialize
+  // "eyJ2ZXJzaW9uI..." in this LazyAst's source_mapping_url field. We'll attach
+  // the "data:application/json;base64," prefix during deserialization.
   string source_mapping_url = 3;
 }
 
@@ -64,8 +68,8 @@ message NonLazyAst {
   AstNode script = 1;
   // 1-based index into TypedAst::source_file_pool
   uint32 source_file = 2;
-  // The URL for a source map associated with this source file (i.e. the URL
-  // specified in "//# sourceMappingURL=" comments).
+  // The encoded source map taken from the inline sourcemap comment
+  // (base64-encoded "data url" stored in `//# sourceMappingURL=` comment).
   string source_mapping_url = 3;
 }
 

test/com/google/javascript/jscomp/SourceMapResolverTest.java

@@ -116,4 +116,14 @@ public void testIntegration() {
             SourceFile.fromCode("somePath/hello.js", ""), url, false);
     assertThat(s.getName()).isEqualTo("somePath/relative/path/to/sourcemap/hello.js.map");
   }
+
+  @Test
+  public void testAddBase64PrefixToEncodedSourceMap() {
+    String base64Prefix = "data:application/json;base64,";
+    String encodedSourceMap =
+        "eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9vLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZm9vLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBO0lBR0UsV0FBWSxLQUFhO1FBQ3ZCLElBQUksQ0FBQyxDQUFDLEdBQUcsS0FBSyxDQUFDO0lBQ2pCLENBQUM7SUFDSCxRQUFDO0FBQUQsQ0FBQyxBQU5ELElBTUM7QUFFRCxPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMifQ==";
+    String base64EncodedSourceMap =
+        SourceMapResolver.addBase64PrefixToEncodedSourceMap(encodedSourceMap);
+    assertThat(base64EncodedSourceMap).isEqualTo(base64Prefix + encodedSourceMap);
+  }
 }

test/com/google/javascript/jscomp/serialization/SerializeAndDeserializeAstTest.java

@@ -404,8 +404,33 @@ public void testEsModule() {
         });
   }
 
+  private static final String BASE64_PREFIX = "data:application/json;base64,";
+  private static final String ENCODED_SOURCE_MAP =
+      "eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9vLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZm9vLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBO0lBR0UsV0FBWSxLQUFhO1FBQ3ZCLElBQUksQ0FBQyxDQUFDLEdBQUcsS0FBSyxDQUFDO0lBQ2pCLENBQUM7SUFDSCxRQUFDO0FBQUQsQ0FBQyxBQU5ELElBTUM7QUFFRCxPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMifQ==";
+
   @Test
-  public void testSourceMaps() {
+  public void testInlineSourceMaps() {
+    String sourceMapTestCode =
+        lines(
+            "var X = (function () {",
+            "    function X(input) {",
+            "        this.y = input;",
+            "    }",
+            "    return X;",
+            "}());");
+    String sourceMappingURLComment = "//# sourceMappingURL=" + BASE64_PREFIX + ENCODED_SOURCE_MAP;
+    ;
+    String code = sourceMapTestCode + "\n" + sourceMappingURLComment;
+
+    Result result = testAndReturnResult(srcs(code), expected(code));
+    assertThat(result.compiler.getBase64SourceMapContents("testcode"))
+        .isEqualTo(ENCODED_SOURCE_MAP);
+  }
+
+  @Test
+  public void testSourceMapsInSeparateMapFiles() {
+    // Sourcemap URLs should be a base64 encoded data url, not the name of a .js.map file.
+    // If we see a .js.map file, we will not serialize it.
     String sourceMapTestCode =
         lines(
             "var X = (function () {",
@@ -419,7 +444,7 @@ public void testSourceMaps() {
     String code = sourceMapTestCode + "\n" + sourceMappingURLComment;
 
     Result result = testAndReturnResult(srcs(code), expected(code));
-    assertThat(result.compiler.getInputSourceMappingURL("testcode")).isEqualTo(sourceMappingURL);
+    assertThat(result.compiler.getBase64SourceMapContents("testcode")).isEqualTo(null);
   }
 
   @Test
@@ -433,13 +458,13 @@ public void testSourceMapsWithoutResolvingSourceMapAnnotations() {
             "    }",
             "    return X;",
             "}());");
-    String sourceMappingURL = "foo.js.map";
-    String sourceMappingURLComment = "//# sourceMappingURL=" + sourceMappingURL;
+    String sourceMappingURLComment = "//# sourceMappingURL=" + BASE64_PREFIX + ENCODED_SOURCE_MAP;
+    ;
     String code = sourceMapTestCode + "\n" + sourceMappingURLComment;
 
     Result result = testAndReturnResult(srcs(code), expected(code));
-    // Input source map not registered because `resolveSourceMapAnnotations = false`
-    assertThat(result.compiler.getInputSourceMappingURL("testcode")).isNull();
+    // Source map not registered because `resolveSourceMapAnnotations = false`
+    assertThat(result.compiler.getBase64SourceMapContents("testcode")).isNull();
   }
 
   @Test
@@ -453,23 +478,22 @@ public void testSourceMapsWithoutParsingInlineSourceMaps() {
             "    }",
             "    return X;",
             "}());");
-    String sourceMappingURL = "foo.js.map";
-    String sourceMappingURLComment = "//# sourceMappingURL=" + sourceMappingURL;
+    String sourceMappingURLComment = "//# sourceMappingURL=" + BASE64_PREFIX + ENCODED_SOURCE_MAP;
+    ;
     String code = sourceMapTestCode + "\n" + sourceMappingURLComment;
 
     Result result = testAndReturnResult(srcs(code), expected(code));
-    // Input source map is registered when `parseInlineSourceMaps = false`, but we won't try to
+    // Source map is registered when `parseInlineSourceMaps = false`, but we won't try to
     // parse it as a Base64 encoded source map.
-    assertThat(result.compiler.getInputSourceMappingURL("testcode")).isEqualTo(sourceMappingURL);
+    assertThat(result.compiler.getBase64SourceMapContents("testcode")).isEqualTo(null);
   }
 
   @Test
   public void testConfiguredDirectorySourceMaps() {
     // We do not allow the TypeScript compiler to set "compilerOptions.sourceRoot" (option to
     // configure a directory to store
-    // sourcemaps). Sourcemaps (.js.map) files are placed next to the .js files.
-    // This means sourcemap URLs should be the name of the sourcemap file, not a path to the
-    // sourcemap file. If we see a path, we will serialize only the name of the sourcemap file.
+    // sourcemaps). Sourcemap URLs should be a base64 encoded data url, not a path to the
+    // sourcemap file. If we see a path, we will not serialize anything.
     String sourceMapTestCode =
         lines(
             "var X = (function () {",
@@ -483,7 +507,7 @@ public void testConfiguredDirectorySourceMaps() {
     String code = sourceMapTestCode + "\n" + sourceMappingURLComment;
 
     Result result = testAndReturnResult(srcs(code), expected(code));
-    assertThat(result.compiler.getInputSourceMappingURL("testcode")).isEqualTo("foo.js.map");
+    assertThat(result.compiler.getBase64SourceMapContents("testcode")).isEqualTo(null);
   }
 
   @Test

test/com/google/javascript/jscomp/serialization/SerializeTypedAstPassTest.java

@@ -425,7 +425,9 @@ public void serializesExternProperties() throws IOException {
   }
 
   @Test
-  public void serializesSourceMappingURL() throws InvalidProtocolBufferException {
+  public void serializeInlineSourceMappingURL() throws InvalidProtocolBufferException {
+    // We want TypedAST to support inline source maps (which are input source maps passed
+    // by embedding a `//# sourceMappingURL=<url>` where <url> is a base64-encoded "data url").
     String sourceMapTestCode =
         lines(
             "var X = (function () {",
@@ -435,20 +437,23 @@ public void serializesSourceMappingURL() throws InvalidProtocolBufferException {
             "    return X;",
             "}());");
 
-    String code = sourceMapTestCode + "\n//# sourceMappingURL=foo.js.map";
+    String base64Prefix = "data:application/json;base64,";
+    String encodedSourceMap =
+        "eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9vLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZm9vLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBO0lBR0UsV0FBWSxLQUFhO1FBQ3ZCLElBQUksQ0FBQyxDQUFDLEdBQUcsS0FBSyxDQUFDO0lBQ2pCLENBQUM7SUFDSCxRQUFDO0FBQUQsQ0FBQyxBQU5ELElBTUM7QUFFRCxPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMifQ==";
+
+    String code = sourceMapTestCode + "\n//# sourceMappingURL=" + base64Prefix + encodedSourceMap;
 
     SerializationResult result = compile(code);
 
     LazyAst lazyAst = result.ast.getCodeAstList().get(0);
     String sourceMappingURL = lazyAst.getSourceMappingUrl();
 
-    assertThat(sourceMappingURL).isEqualTo("foo.js.map");
+    assertThat(sourceMappingURL).isEqualTo(encodedSourceMap); // Do not serizile the base-64 prefix.
   }
 
   @Test
-  public void doesNotSerializeInlineSourceMappingURL() throws InvalidProtocolBufferException {
-    // We do not want TypedAST to support inline source maps (which are input source maps passed
-    // by embedding a `//# sourceMappingURL=<url>` where <url> is a base64-encoded "data url").
+  public void doesNotSerializeInputSourceMappingURL() throws InvalidProtocolBufferException {
+    // We do not want TypedAST to support input source maps (source maps in separate files).
     String sourceMapTestCode =
         lines(
             "var X = (function () {",
@@ -458,17 +463,14 @@ public void doesNotSerializeInlineSourceMappingURL() throws InvalidProtocolBuffe
             "    return X;",
             "}());");
 
-    String base64EncodedSourceMappingURL =
-        "data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9vLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdGVzdC9mb28udHMiXSwic291cmNlc0NvbnRlbnQiOlsidmFyIEEgPSAoZnVuY3Rpb24gKCkge1xuICAgIGZ1bmN0aW9uIEEoaW5wdXQpIHtcbiAgICAgICAgdGhpcy5hID0gaW5wdXQ7XG4gICAgfVxuICAgIHJldHVybiBBO1xufSgpKTtcbmNvbnNvbGUubG9nKG5ldyBBKDEpKTsiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7SUFHRSxXQUFZLEtBQWE7UUFDdkIsSUFBSSxDQUFDLENBQUMsR0FBRyxLQUFLLENBQUM7SUFDakIsQ0FBQztJQUNILFFBQUM7QUFBRCxDQUFDLEFBTkQsSUFNQztBQUVELE9BQU8sQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyJ9";
-
-    String code = sourceMapTestCode + "\n//# sourceMappingURL=" + base64EncodedSourceMappingURL;
+    String code = sourceMapTestCode + "\n//# sourceMappingURL=foo.js.map";
 
     SerializationResult result = compile(code);
 
     LazyAst lazyAst = result.ast.getCodeAstList().get(0);
     String sourceMappingURL = lazyAst.getSourceMappingUrl();
 
-    assertThat(sourceMappingURL).isEmpty(); // Do not serizile this base-64 sourceMappingURL.
+    assertThat(sourceMappingURL).isEmpty();
   }
 
   private AstNode compileToAstNode(String source) {