Output JS source mapping file (instead of .txtmap).

Author: yurydelendik

check.py

@@ -177,19 +177,19 @@ def do_asm2wasm_test():
 
         # verify debug info
         if 'debugInfo' in asm:
-          txtmap = 'a.wasm.txtmap'
-          cmd += ['--binarymap-file', txtmap,
-                  '--binarymap-url', txtmap + '.map',
+          jsmap = 'a.wasm.map'
+          cmd += ['--binarymap-file', jsmap,
+                  '--binarymap-url', 'http://example.org/' + jsmap,
                   '-o', 'a.wasm']
           run_command(cmd)
-          if not os.path.isfile(txtmap):
-            fail_with_error('Debug info map not created: %s' % txtmap)
-          with open(wasm + '.txtmap', 'rb') as expected:
-            with open(txtmap, 'rb') as actual:
+          if not os.path.isfile(jsmap):
+            fail_with_error('Debug info map not created: %s' % jsmap)
+          with open(wasm + '.map', 'rb') as expected:
+            with open(jsmap, 'rb') as actual:
               fail_if_not_identical(actual.read(), expected.read())
           with open('a.wasm', 'rb') as binary:
             url_section_name = bytearray([16]) + bytearray('sourceMappingURL')
-            payload = txtmap + '.map'
+            payload = 'http://example.org/' + jsmap
             assert len(payload) < 256, 'name too long'
             url_section_contents = bytearray([len(payload)]) + bytearray(payload)
             print url_section_name
@@ -252,6 +252,8 @@ def do_asm2wasm_test():
     print '..', t
     t = os.path.join(options.binaryen_test, t)
     cmd = WASM_DIS + [t]
+    if os.path.isfile(t + '.map'): cmd += ['-bm', t + '.map']
+
     actual = run_command(cmd)
 
     with open(t + '.fromBinary') as f:

src/parsing.h

@@ -193,6 +193,25 @@ struct ParseException {
   }
 };
 
+struct MapParseException {
+  std::string text;
+
+  MapParseException() : text("unknown parse error") {}
+  MapParseException(std::string text) : text(text) {}
+
+  void dump(std::ostream& o) {
+    Colors::magenta(o);
+    o << "[";
+    Colors::red(o);
+    o << "map parse exception: ";
+    Colors::green(o);
+    o << text;
+    Colors::magenta(o);
+    o << "]";
+    Colors::normal(o);
+  }
+};
+
 // Helper for parsers that may not have unique label names. This transforms
 // the names into unique ones, as required by Binaryen IR.
 struct UniqueNameMapper {

src/tools/wasm-dis.cpp

@@ -64,6 +64,9 @@ int main(int argc, const char *argv[]) {
   } catch (ParseException& p) {
     p.dump(std::cerr);
     Fatal() << "error in parsing wasm binary";
+  } catch (MapParseException& p) {
+    p.dump(std::cerr);
+    Fatal() << "error in parsing wasm source mapping";
   }
 
   if (options.debug) std::cerr << "Printing..." << std::endl;

src/wasm-binary.h

@@ -593,6 +593,10 @@ class WasmBinaryWriter : public Visitor<WasmBinaryWriter, void> {
   void writeSourceMapUrl();
   void writeSymbolMap();
 
+  void writeBinaryMapProlog();
+  void writeBinaryMapEpilog();
+  void writeDebugLocation(size_t offset, const Function::DebugLocation& loc);
+
   // helpers
   void writeInlineString(const char* name);
   void writeInlineBuffer(const char* data, size_t size);
@@ -616,16 +620,15 @@ class WasmBinaryWriter : public Visitor<WasmBinaryWriter, void> {
   void recurse(Expression*& curr);
   std::vector<Name> breakStack;
   Function::DebugLocation lastDebugLocation;
+  size_t lastBytecodeOffset;
 
   void visit(Expression* curr) {
     if (binaryMap && currFunction) {
       // Dump the binaryMap debug info
       auto& debugLocations = currFunction->debugLocations;
       auto iter = debugLocations.find(curr);
       if (iter != debugLocations.end() && iter->second != lastDebugLocation) {
-        lastDebugLocation = iter->second;
-        auto fileName = wasm->debugInfoFileNames[iter->second.fileIndex];
-        *binaryMap << o.size() << ":" << fileName << ":" << iter->second.lineNumber << ":" << iter->second.columnNumber << '\n';
+        writeDebugLocation(o.size(), iter->second);
       }
     }
     Visitor<WasmBinaryWriter>::visit(curr);
@@ -767,11 +770,11 @@ class WasmBinaryBuilder {
   // Debug information reading helpers
   void setDebugLocations(std::istream* binaryMap_) {
       binaryMap = binaryMap_;
-      readNextDebugLocation();
   }
   Function::DebugLocation debugLocation;
   std::unordered_map<std::string, Index> debugInfoFileIndices;
   void readNextDebugLocation();
+  void readBinaryMapHeader();
 
   // AST reading
   int depth = 0; // only for debugging

src/wasm/wasm-binary.cpp

@@ -33,6 +33,9 @@ void WasmBinaryWriter::prepare() {
 
 void WasmBinaryWriter::write() {
   writeHeader();
+  if (binaryMap) {
+    writeBinaryMapProlog();
+  }
 
   writeTypes();
   writeImports();
@@ -49,6 +52,9 @@ void WasmBinaryWriter::write() {
   if (binaryMap) writeSourceMapUrl();
   if (symbolMap.size() > 0) writeSymbolMap();
 
+  if (binaryMap) {
+    writeBinaryMapEpilog();
+  }
   finishUp();
 }
 
@@ -238,7 +244,6 @@ void WasmBinaryWriter::writeFunctions() {
     size_t start = o.size();
     Function* function = wasm->functions[i].get();
     currFunction = function;
-    lastDebugLocation = {0, 0, 0};
     mappedLocals.clear();
     numLocalsByType.clear();
     if (debug) std::cerr << "writing" << function->name << std::endl;
@@ -445,6 +450,50 @@ void WasmBinaryWriter::writeSymbolMap() {
   file.close();
 }
 
+void WasmBinaryWriter::writeBinaryMapProlog() {
+  lastDebugLocation = {0, /* lineNumber = */ 1, 0};
+  lastBytecodeOffset = 0;
+  *binaryMap << "{\"version\":3,\"sources\":[";
+  for (size_t i = 0; i < wasm->debugInfoFileNames.size(); i++) {
+    if (i > 0) *binaryMap << ",";
+    // TODO respect JSON string encoding, e.g. quotes and control chars.
+    *binaryMap << "\"" << wasm->debugInfoFileNames[i] << "\"";
+  }
+  *binaryMap << "],\"names\":[],\"mappings\":\"";
+}
+
+void WasmBinaryWriter::writeBinaryMapEpilog() {
+  *binaryMap << "\"}";
+}
+
+static void writeBase64VLQ(std::ostream& out, int32_t n) {
+  uint32_t value = n >= 0 ? n << 1 : ((-n) << 1) | 1;
+  while (1) {
+    uint32_t digit = value & 0x1F;
+    value >>= 5;
+    if (!value) {
+      // last VLQ digit -- base64 codes 'A'..'Z', 'a'..'f'
+      out << char(digit < 26 ? 'A' + digit : 'a' + digit - 26);
+      break;
+    }
+    // more VLG digit will follow -- add continuation bit (0x20),
+    // base64 codes 'g'..'z', '0'..'9', '+', '/'
+    out << char(digit < 20 ? 'g' + digit : digit < 30 ? '0' + digit - 20 : digit == 30 ? '+' : '/');
+  }
+}
+
+void WasmBinaryWriter::writeDebugLocation(size_t offset, const Function::DebugLocation& loc) {
+  if (lastBytecodeOffset > 0) {
+    *binaryMap << ",";
+  }
+  writeBase64VLQ(*binaryMap, int32_t(offset - lastBytecodeOffset));
+  writeBase64VLQ(*binaryMap, int32_t(loc.fileIndex - lastDebugLocation.fileIndex));
+  writeBase64VLQ(*binaryMap, int32_t(loc.lineNumber - lastDebugLocation.lineNumber));
+  writeBase64VLQ(*binaryMap, int32_t(loc.columnNumber - lastDebugLocation.columnNumber));
+  lastDebugLocation = loc;
+  lastBytecodeOffset = offset;
+}
+
 void WasmBinaryWriter::writeInlineString(const char* name) {
   int32_t size = strlen(name);
   o << U32LEB(size);
@@ -949,6 +998,7 @@ static Name RETURN_BREAK("binaryen|break-to-return");
 void WasmBinaryBuilder::read() {
 
   readHeader();
+  readBinaryMapHeader();
 
   // read sections until the end
   while (more()) {
@@ -1402,43 +1452,134 @@ void WasmBinaryBuilder::readExports() {
   }
 }
 
-void WasmBinaryBuilder::readNextDebugLocation() {
-  if (binaryMap) {
-    std::string line;
-    while (std::getline(*binaryMap, line)) {
-      auto pos = line.begin();
-      while (pos < line.end() && pos[0] != ':') pos++;
-      if (pos == line.end()) continue;
-      uint32_t position = atoi(std::string(line.begin(), pos).c_str());
-      auto filenameStart = ++pos;
-      while (pos < line.end() && pos[0] != ':') pos++;
-      if (pos == line.end()) continue;
-      std::string file(filenameStart, pos);
-      auto iter = debugInfoFileIndices.find(file);
-      if (iter == debugInfoFileIndices.end()) {
-        Index index = wasm.debugInfoFileNames.size();
-        wasm.debugInfoFileNames.push_back(file);
-        debugInfoFileIndices[file] = index;
+static int32_t readBase64VLQ(std::istream& in) {
+  uint32_t value = 0;
+  uint32_t shift = 0;
+  while (1) {
+    char ch = in.get();
+    if (ch == EOF)
+      throw MapParseException("unexpected EOF in the middle of VLQ");
+    if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch < 'g')) {
+      // last number digit
+      uint32_t digit = ch < 'a' ? ch - 'A' : ch - 'a' + 26;
+      value |= digit << shift;
+      break;
+    }
+    if (!(ch >= 'g' && ch <= 'z') && !(ch >= '0' && ch <= '9') &&
+        ch != '+' && ch != '/') {
+      throw MapParseException("invalid VLQ digit");
+    }
+    uint32_t digit = ch > '9' ? ch - 'g' : (ch >= '0' ? ch - '0' + 20 : (ch == '+' ? 30 : 31));
+    value |= digit << shift;
+    shift += 5;
+  }
+  return value & 1 ? -int32_t(value >> 1) : int32_t(value >> 1);
+}
+
+void WasmBinaryBuilder::readBinaryMapHeader() {
+  if (!binaryMap) {
+    return;
+  }
+  auto maybeReadChar = [&](char expected) {
+    if (binaryMap->peek() != expected)
+      return false;
+    binaryMap->get();
+    return true;
+  };
+  auto mustReadChar = [&](char expected) {
+    if (binaryMap->get() != expected)
+      throw MapParseException("Unexpected char");
+  };
+  auto findField = [&](const char* name, size_t len) {
+    bool matching = false;
+    size_t pos;
+    while (1) {
+      int ch = binaryMap->get();
+      if (ch == EOF) return false;
+      if (ch == '\"') {
+        matching = true;
+        pos = 0;
+      } else if (matching && name[pos] == ch) {
+        ++pos;
+        if (pos == len) {
+          if (maybeReadChar('\"')) break; // found field
+        }
+      } else {
+        matching = false;
       }
-      uint32_t fileIndex = debugInfoFileIndices[file];
-      auto lineNumberStart = ++pos;
-      while (pos < line.end() && pos[0] != ':') pos++;
-      if (pos == line.end()) {
-        // old format
-        uint32_t lineNumber = atoi(std::string(lineNumberStart, line.end()).c_str());
-        nextDebugLocation = {position, {fileIndex, lineNumber, 0}};
-        return;
+    }
+    mustReadChar(':');
+    return true;
+  };
+  auto readString = [&](std::string& str) {
+    std::vector<char> vec;
+    mustReadChar('\"');
+    if (!maybeReadChar('\"')) {
+      while (1) {
+        int ch = binaryMap->get();
+        if (ch == EOF)
+          throw MapParseException("unexpected EOF in the middle of string");
+        if (ch == '\"') break;
+        vec.push_back(ch);
       }
-      uint32_t lineNumber = atoi(std::string(lineNumberStart, pos).c_str());
-      auto columnNumberStart = ++pos;
-      uint32_t columnNumber = atoi(std::string(columnNumberStart, line.end()).c_str());
-
-      nextDebugLocation = {position, {fileIndex, lineNumber, columnNumber}};
-      return;
     }
+    str = std::string(vec.begin(), vec.end());
+  };
+
+  if (!findField("sources", strlen("sources")))
+    throw MapParseException("cannot find the sources field in map");
+  mustReadChar('[');
+  if (!maybeReadChar(']')) {
+    do {
+      std::string file;
+      readString(file);
+      Index index = wasm.debugInfoFileNames.size();
+      wasm.debugInfoFileNames.push_back(file);
+      debugInfoFileIndices[file] = index;
+    } while (maybeReadChar(','));
+    mustReadChar(']');
+  }
+
+  if (!findField("mappings", strlen("mappings")))
+    throw MapParseException("cannot find the mappings field in map");
+  mustReadChar('\"');
+  if (maybeReadChar('\"')) { // empty mappings
+    nextDebugLocation.first = 0;
+    return;
+  }
+  // read first debug location
+  uint32_t position = readBase64VLQ(*binaryMap);
+  uint32_t fileIndex = readBase64VLQ(*binaryMap);
+  uint32_t lineNumber = readBase64VLQ(*binaryMap) + 1; // adjust zero-based line number
+  uint32_t columnNumber = readBase64VLQ(*binaryMap);
+  nextDebugLocation = {position, {fileIndex, lineNumber, columnNumber}};
+}
+
+void WasmBinaryBuilder::readNextDebugLocation() {
+  if (!binaryMap) {
+    return;
+  }
+  char ch;
+  *binaryMap >> ch;
+  if (ch == '\"') { // end of records
     nextDebugLocation.first = 0;
+    return;
   }
+  if (ch != ',')
+    throw MapParseException("Unexpected delimiter");
+
+  int32_t positionDelta = readBase64VLQ(*binaryMap);
+  uint32_t position = nextDebugLocation.first + positionDelta;
+  int32_t fileIndexDelta = readBase64VLQ(*binaryMap);
+  uint32_t fileIndex = nextDebugLocation.second.fileIndex + fileIndexDelta;
+  int32_t lineNumberDelta = readBase64VLQ(*binaryMap);
+  uint32_t lineNumber = nextDebugLocation.second.lineNumber + lineNumberDelta;
+  int32_t columnNumberDelta = readBase64VLQ(*binaryMap);
+  uint32_t columnNumber = nextDebugLocation.second.columnNumber + columnNumberDelta;
+
+  nextDebugLocation = {position, {fileIndex, lineNumber, columnNumber}};
 }
+
 Expression* WasmBinaryBuilder::readExpression() {
   assert(depth == 0);
   processExpressions();