[ZEPPELIN-6634] Download service prepends a UTF-8 BOM to JSON files, breaking standard parsers

zeppelin-web-angular/src/app/services/save-as.service.spec.ts

@@ -0,0 +1,76 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { SaveAsService } from './save-as.service';
+
+/**
+ * Returns the bytes the service handed to createObjectURL. Blob.text() decodes and strips a
+ * leading BOM, so the raw bytes are the only way to tell whether one was written.
+ */
+async function downloadedBytes(saveAs: () => void): Promise<Uint8Array> {
+  let saved: Blob | undefined;
+  // the service goes through window.URL, which is not the same object vi.stubGlobal replaces
+  vi.spyOn(window.URL, 'createObjectURL').mockImplementation((blob: Blob | MediaSource) => {
+    saved = blob as Blob;
+    return 'blob:url';
+  });
+  vi.spyOn(window.URL, 'revokeObjectURL').mockImplementation(() => undefined);
+
+  saveAs();
+
+  expect(saved).toBeDefined();
+  return new Uint8Array(await (saved as Blob).arrayBuffer());
+}
+
+const UTF8_BOM = [0xef, 0xbb, 0xbf];
+
+function hasBom(bytes: Uint8Array): boolean {
+  return UTF8_BOM.every((byte, index) => bytes[index] === byte);
+}
+
+describe('SaveAsService', () => {
+  let service: SaveAsService;
+
+  beforeEach(() => {
+    service = new SaveAsService();
+    // jsdom has no navigation, so the anchor click must not actually do anything
+    vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => undefined);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('leaves a JSON export without a BOM', async () => {
+    const content = '{"paragraphs":[{"text":"한글 テスト 中文 🎉"}]}';
+
+    const bytes = await downloadedBytes(() => service.saveAs(content, 'note', 'zpln'));
+
+    // a BOM here makes strict JSON parsers such as Python's json or nbformat reject the file
+    expect(hasBom(bytes)).toBe(false);
+    const text = new TextDecoder().decode(bytes);
+    expect(text).toBe(content);
+    expect(JSON.parse(text)).toEqual(JSON.parse(content));
+  });
+
+  it('prepends a BOM when the caller asks for one', async () => {
+    const content = 'name,value\n한글,1\n';
+
+    const bytes = await downloadedBytes(() => service.saveAs(content, 'result', 'csv', true));
+
+    // Excel needs the BOM to read the CSV as UTF-8 (ZEPPELIN-672)
+    expect(hasBom(bytes)).toBe(true);
+    expect(new TextDecoder().decode(bytes.subarray(UTF8_BOM.length))).toBe(content);
+  });
+});

zeppelin-web-angular/src/app/services/save-as.service.ts

@@ -16,11 +16,18 @@ import { Injectable } from '@angular/core';
   providedIn: 'root'
 })
 export class SaveAsService {
-  saveAs(content: string, filename: string, extension: string) {
+  /**
+   * @param bom prepends a UTF-8 BOM so Excel reads a CSV/TSV export as UTF-8 (ZEPPELIN-672).
+   *            JSON formats must leave it off: the JSON spec disallows a BOM and strict parsers
+   *            such as Python's json or nbformat refuse the file.
+   */
+  saveAs(content: string, filename: string, extension: string, bom = false) {
     const BOM = '\uFEFF';
     const fileName = `${filename}.${extension}`;
     const binaryData = [];
-    binaryData.push(BOM);
+    if (bom) {
+      binaryData.push(BOM);
+    }
     binaryData.push(content);
     const blob = new Blob(binaryData, { type: 'octet/stream' });
     const url = window.URL.createObjectURL(blob);

zeppelin-web/src/app/notebook/paragraph/result/result.controller.js

@@ -911,7 +911,8 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
     } else if (delimiter === ',') {
       extension = 'csv';
     }
-    saveAsService.saveAs(dsv, exportedFileName, extension);
+    // CSV and TSV keep the BOM so Excel reads them as UTF-8 (ZEPPELIN-672)
+    saveAsService.saveAs(dsv, exportedFileName, extension, true);
   };
 
   $scope.copyToClipboard = function(delimiter) {

zeppelin-web/src/app/notebook/save-as/save-as.service.js

@@ -17,12 +17,19 @@ angular.module('zeppelinWebApp').service('saveAsService', SaveAsService);
 function SaveAsService(browserDetectService) {
   'ngInject';
 
-  this.saveAs = function(content, filename, extension) {
+  /**
+   * @param {boolean} [bom] prepends a UTF-8 BOM so Excel reads a CSV/TSV export as UTF-8
+   *   (ZEPPELIN-672). JSON formats must leave it off: the JSON spec disallows a BOM and strict
+   *   parsers such as Python's json or nbformat refuse the file.
+   */
+  this.saveAs = function(content, filename, extension, bom) {
     let BOM = '\uFEFF';
     if (browserDetectService.detectIE()) {
       angular.element('body').append('<iframe id="SaveAsId" style="display: none"></iframe>');
       let frameSaveAs = angular.element('body > iframe#SaveAsId')[0].contentWindow;
-      content = BOM + content;
+      if (bom) {
+        content = BOM + content;
+      }
       frameSaveAs.document.open('text/json', 'replace');
       frameSaveAs.document.write(content);
       frameSaveAs.document.close();
@@ -40,7 +47,9 @@ function SaveAsService(browserDetectService) {
     } else {
       const fileName = filename + '.' + extension;
       let binaryData = [];
-      binaryData.push(BOM);
+      if (bom) {
+        binaryData.push(BOM);
+      }
       binaryData.push(content);
       let blob = new Blob(binaryData, {type: 'octet/stream'});
       const url = window.URL.createObjectURL(blob);