Merge pull request #154 from gregolsky/perf

Improve load() performance + minor fixes

Author: gregolsky

.travis.yml

@@ -5,7 +5,8 @@ node_js:
   - "10"
 env:
   matrix:
-    - RAVENDB_TEST_SERVER_PATH=./RavenDB/Server/Raven.Server
+    - RAVENDB_SERVER_DOWNLOAD="https://hibernatingrhinos.com/downloads/RavenDB%20for%20Linux%20x64/latest?buildType=stable&version=4.0"
+      RAVENDB_TEST_SERVER_PATH=./RavenDB/Server/Raven.Server
       RAVENDB_TEST_SERVER_CERTIFICATE_PATH=./certs/server.pfx
       RAVENDB_TEST_CLIENT_CERT_PASSPHRASE=client11
       RAVENDB_TEST_CLIENT_CERT_PATH=./certs/nodejs.pem
@@ -14,7 +15,7 @@ env:
 before_install:
   - sudo apt-get -qq update
   - sudo apt-get install -y libunwind8 wget libicu52 libssl-dev curl unzip gettext libcurl4-openssl-dev zlib1g uuid-dev bzip2 openssl
-  - wget -O RavenDB.tar.bz2 https://hibernatingrhinos.com/downloads/RavenDB%20for%20Linux%20x64/latest?buildType=nightly
+  - wget -O RavenDB.tar.bz2 "$RAVENDB_SERVER_DOWNLOAD" 
   - tar xjf RavenDB.tar.bz2
   - mkdir certs; pushd certs
   - openssl genrsa -out ca.key 2048

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "ravendb",
-  "version": "4.0.3",
+  "version": "4.0.4",
   "description": "RavenDB client for Node.js",
   "files": [
     "dist/"
@@ -76,7 +76,6 @@
     "bluebird": "^3.4.7",
     "change-case": "^3.0.2",
     "md5-hex": "^2.0.0",
-    "merge2": "^1.2.2",
     "moment": "^2.22.0",
     "pluralize": "^4.0.0",
     "qs": "^6.5.2",
@@ -86,7 +85,6 @@
     "semaphore": "^1.1.0",
     "stream-json": "^1.1.1",
     "string-builder": "^0.1.6",
-    "through2": "^2.0.3",
     "util.promisify": "^1.0.0",
     "uuid": "^3.2.1",
     "verror": "^1.10.0",

package.json

@@ -0,0 +1,8 @@
+import { DocumentStore } from "../src";
+
+// tslint:disable-next-line:no-var-requires
+const settings = require("./settings.json");
+
+export function getStore() {
+    return new DocumentStore(settings.urls, settings.database);
+}

perf/common.js

@@ -0,0 +1,23 @@
+{
+    "Results": [
+        {
+            "TestKey": "TestValue",
+            "NumberKey": 1,
+            "@metadata": {
+                "@collection": "hahahah",
+                "Raven-Node-Type": "TypeName",
+                "@nested-object-types": {
+                    "start": "date"
+                },
+                "@attachments": [
+                    { 
+                        "Name": "test.png",
+                        "Hash": "asdij%*(J#JSA",
+                        "ContentType": "aaaa"
+                    }
+                ]
+            }
+        }
+    ],
+    "Includes": {}
+}

perf/data/load_data.json

@@ -0,0 +1,100 @@
+const { DocumentStore } = require("../dist");
+const { GetDocumentsCommand } = require("../dist/Documents/Commands/GetDocumentsCommand");
+const { TransformKeysJsonStream } = require("../dist/Mapping/Json/Streams/TransformKeysJsonStream");
+const fs = require('fs');
+const parser = require("stream-json")
+const pick = require("stream-json/filters/Pick")
+const { ignore } = require("stream-json/filters/Ignore")
+const { replace } = require("stream-json/filters/Replace")
+const streamArray = require("stream-json/streamers/StreamArray")
+const streamObject = require("stream-json/streamers/StreamObject")
+const streamValues = require("stream-json/streamers/StreamValues")
+const StreamUtil = require("../dist/Utility/StreamUtil");
+const stream = require("readable-stream");
+const Asm = require('stream-json/Assembler');
+
+const store = new DocumentStore("http://localhost:8080", "Perf");
+store.initialize();
+
+async function bench(name, attempts, run) {
+    const benchName = `${name} x${ attempts }`;
+    console.time(benchName);
+    for (let n = 0; n < attempts; n++) {
+        await run();
+    }
+    console.timeEnd(benchName);
+}
+
+(async function main() {
+    {
+        const name = "4.0.4-load-full-pipeline";
+        await bench(name, 10, loadPipeline);
+        await bench(name, 50, loadPipeline);
+        await bench(name, 100, loadPipeline);
+    }
+
+    // {
+    //     const name = "stream-json-with-proper-casing";
+    //     // enhancedStreamJson();
+    //     await bench(name, 10, enhancedStreamJson);
+    //     await bench(name, 50, enhancedStreamJson);
+    //     await bench(name, 100, enhancedStreamJson);
+    // }
+
+    store.dispose();
+}());
+
+async function loadPipeline() {
+    const dataStream = fs.createReadStream("./data/load_data.json");
+    let body;
+    const results = await GetDocumentsCommand
+        .parseDocumentsResultResponseAsync(dataStream, store.conventions, _ => body = _);
+}
+
+async function rawStreamJson() {
+    const dataStream = fs.createReadStream("./data/load_data.json");
+    const parserStream = parser();
+    const asm = Asm.connectTo(parserStream);
+    const donePromise = new Promise(resolve => {
+        asm.on('done', asm => {
+            resolve();
+        });
+    });
+    await StreamUtil.pipelineAsync([
+        dataStream,
+        parserStream,
+        ignore({ filter: "asasas" })
+    ]);
+    await donePromise;
+}
+
+
+
+async function enhancedStreamJson() {
+    const dataStream = fs.createReadStream("./data/load_data.json");
+    const streams = [
+        dataStream,
+        parser({
+            packKeys: true,
+            packStrings: true,
+            packValues: true,
+            packNumbers: true,
+            streamNumbers: false,
+            streamValues: false,
+            streamKeys: false,
+            streamStrings: false
+        }),
+        new TransformKeysJsonStream({
+            getCurrentTransform: buildEntityKeysTransform("camel")
+        })
+    ];
+    const asm = Asm.connectTo(streams[streams.length - 1]);
+    const donePromise = new Promise(resolve => {
+        asm.on('done', asm => {
+            resolve(asm.current);
+        });
+    });
+    await StreamUtil.pipelineAsync(streams);
+    const result = await donePromise;
+    // console.log(JSON.stringify(result, null, 2));
+}