Preparations for next version release

Author: gignupg

README.md

@@ -103,7 +103,7 @@ Now that you've loaded the module, you can [start using it](#usage-1).
 
 ##### Usage
 
-The `<script>` tag exposes the `languageEncoding` function to everything in the DOM located beneath it. When you call it and pass in the file that you want to analyze, it'll return a Promise that you can use to retrieve the encoding, language and confidenc score as shown in the example below.
+The `<script>` tag exposes the `languageEncoding` function to everything in the DOM located beneath it. When you call it and pass in the file that you want to analyze, it'll return a Promise that you can use to retrieve the encoding, language and confidence score as shown in the example below.
 
 ```js
 // app.js
@@ -116,7 +116,7 @@ function inputHandler(e) {
   const file = e.target.files[0];
 
   languageEncoding(file).then((fileInfo) => console.log(fileInfo));
-  // Possible result: { language: english, encoding: UTF-8, confidence: 0.97}
+  // Possible result: { language: english, encoding: UTF-8, confidence: { encoding: 1, language: 1 } }
 }
 ```
 
@@ -143,7 +143,7 @@ function inputHandler(e) {
   const file = e.target.files[0];
 
   languageEncoding(file).then((fileInfo) => console.log(fileInfo));
-  // Possible result: { language: english, encoding: UTF-8, confidence: 0.97}
+  // Possible result: { language: french, encoding: CP1252, confidence: { encoding: 1, language: 0.97 } }
 }
 ```
 
@@ -167,7 +167,7 @@ const languageEncoding = require("detect-file-encoding-and-language");
 const pathToFile = "/home/username/documents/my-text-file.txt";
 
 languageEncoding(pathToFile).then((fileInfo) => console.log(fileInfo));
-// Possible result: { language: japanese, encoding: Shift-JIS, confidence: 1 }
+// Possible result: { language: japanese, encoding: Shift-JIS, confidence: { encoding: 0.94, language: 0.94 } }
 ```
 
 ### In the terminal (CLI)
@@ -184,14 +184,14 @@ Once installed you'll be able to use the command `dfeal` to retrieve the encodin
 
 ```bash
 $ dfeal "/home/user name/Documents/subtitle file.srt"
-# Possible result: { language: french, encoding: CP1252, confidence: 0.99 }
+# Possible result: { language: french, encoding: CP1252, confidence: { encoding: 0.99, language: 0.99 } }
 ```
 
 or without quotation marks, using backslashes to escape spaces:
 
 ```bash
 $ dfeal /home/user\ name/Documents/subtitle\ file.srt
-# Possible result: { language: french, encoding: CP1252, confidence: 0.99 }
+# Possible result: { language: french, encoding: CP1252, confidence: { encoding: 0.97, language: 0.97 } }
 ```
 
 ## Supported Languages
@@ -256,7 +256,7 @@ $ dfeal /home/user\ name/Documents/subtitle\ file.srt
 
 ## Confidence Score
 
-The confidence score ranges from 0 to 1. It is based on the amount of matches that were found for a particular language and the frequency of those matches. If you want to learn more about how it all works, check out the [Wiki entry](https://github.com/gignupg/Detect-File-Encoding-and-Language/wiki)!
+The confidence score ranges from 0 to 1. It's an object that contains two different confidence scores. The language confidence score and the encoding confidence score. Both confidence scores will be the same if the detected encoding is Unicode. Otherwise the confidence score for the language and the encoding is calculated seperately. It is based on the amount of matches that were found for a particular language and the frequency of those matches. If you want to learn more about how it all works, check out the [Wiki entry](https://github.com/gignupg/Detect-File-Encoding-and-Language/wiki)!
 
 ## Known Issues
 

package.json

@@ -1,11 +1,12 @@
 {
   "name": "detect-file-encoding-and-language",
-  "version": "1.7.4",
+  "version": "2.0.1",
   "description": "Charset Detector - Detect the encoding and language of any file - Use it in the browser, with Node.js, or via CLI",
   "main": "src/index-node.js",
   "scripts": {
     "regextest": "node ./testing/regexTester.test.js",
     "test": "node ./testing/language-encoding.test.js",
+    "build": "browserify --standalone ./src/index-browser.js > ./umd/language-encoding.min.js",
     "prepublishOnly": "npm test"
   },
   "browser": "umd/language-encoding.min.js",

src/index-browser.js

@@ -1,35 +1,85 @@
-const checkUTF = require('./components/checkUTF.js');
-const processContent = require('./components/processContent.js');
+const checkUTF = require("./components/checkUTF.js");
+const processContent = require("./components/processContent.js");
+const checkByteOrderMark = require("./components/checkByteOrderMark.js");
 
-module.exports = (file, test) => {
-    return new Promise((resolve, reject) => {
-        const data = {};
-        const utfReader = new FileReader();
+module.exports = (file) => {
+  return new Promise((resolve, reject) => {
+    const fileInfo = {
+      encoding: null,
+      language: null,
+      confidence: {
+        encoding: null,
+        language: null,
+      },
+    };
+    const data = {};
 
-        utfReader.onerror = (err) => {
-            reject(err);
+    // Check the byte order mark!
+    const byteOrderMarkBuffer = new FileReader();
+
+    byteOrderMarkBuffer.onload = () => {
+      const uInt8String = new Uint8Array(byteOrderMarkBuffer.result)
+        .slice(0, 4)
+        .join(" ");
+      const byteOrderMark = checkByteOrderMark(uInt8String);
+
+      if (byteOrderMark) {
+        fileInfo.encoding = byteOrderMark;
+        fileInfo.confidence.encoding = 1;
+
+        const byteOrderMarkReader = new FileReader();
+
+        byteOrderMarkReader.onload = () => {
+          data.content = byteOrderMarkReader.result;
+          resolve(processContent(data, fileInfo));
+        };
+
+        byteOrderMarkReader.onerror = (err) => {
+          reject(err);
         };
 
+        byteOrderMarkReader.readAsText(file, fileInfo.encoding);
+      } else {
+        // Read with UTF-8 first, then with ISO-8859-1
+        const utfReader = new FileReader();
+
         utfReader.onload = () => {
-            const utfContent = utfReader.result;
+          const utfContent = utfReader.result;
 
-            data.utf8 = checkUTF(utfContent);
+          const utf8 = checkUTF(utfContent);
 
-            if (data.utf8) {
-                data.content = utfContent;
-                resolve(processContent(data));
+          if (utf8) {
+            fileInfo.encoding = "UTF-8";
+            fileInfo.confidence.encoding = 1;
+          }
 
-            } else {
-                const isoReader = new FileReader();
+          if (utf8) {
+            data.content = utfContent;
+            resolve(processContent(data, fileInfo));
+          } else {
+            const isoReader = new FileReader();
 
-                isoReader.onload = () => {
-                    data.content = isoReader.result;
-                    resolve(processContent(data));
-                };
+            isoReader.onload = () => {
+              data.content = isoReader.result;
+              resolve(processContent(data, fileInfo));
+            };
+
+            isoReader.readAsText(file, "ISO-8859-1");
+          }
+        };
 
-                isoReader.readAsText(file, "ISO-8859-1");
-            }
+        utfReader.onerror = (err) => {
+          reject(err);
         };
+
         utfReader.readAsText(file, "UTF-8");
-    });
-};
+      }
+    };
+
+    byteOrderMarkBuffer.onerror = (err) => {
+      reject(err);
+    };
+
+    byteOrderMarkBuffer.readAsArrayBuffer(file);
+  });
+};

src/index-node.js

@@ -19,7 +19,6 @@ module.exports = (file, test) => {
     // Reading the first four bytes and checking if they coincide with one of the predefined byte order marks.
     const readStream = fs.createReadStream(file, { start: 0, end: 3 });
 
-    // This will wait until we know the readable stream is actually valid before piping
     readStream.on("data", function (buffer) {
       const uInt8Array = new Uint8Array(buffer);
       const uInt8String = uInt8Array.join(" ");