bulk query refactoring (#161)

* bulk query refactoring
* Delete object use secrets (#162)
* bulk_cud.js refactoring (#163)
* Lookup objects use secrets (#164)
* Triggers use secrets (#165)
* polling and query triggers use secrets
* lookupObject use secrets (#167)
* Fix Unit Tests (#166)
* Update dependencies and fix eslint
* fix logging sensitive data
* refactor circleci
* delete mock for platform issue 4527

Author: Olha Virolainen

.circleci/build_slug.sh

@@ -2,7 +2,7 @@ version: 2
 jobs:
   test:
     docker:
-      - image: circleci/node:8-stretch
+      - image: circleci/node:12-stretch
     steps:
       - checkout
       - restore_cache:

circle.yml .circleci/config.ymlcircle.yml renamed to .circleci/config.yml

@@ -1,14 +1,14 @@
 module.exports = {
   env: {
-    browser: true,
     commonjs: true,
     es6: true,
-    mocha: true
+    mocha: true,
   },
   extends: [
     'airbnb-base',
   ],
   rules: {
-    'max-len': ["error", { "code": 150 }]
-  }
+    'no-await-in-loop': 0,
+    'max-len': ['error', { code: 150 }],
+  },
 };

.eslintrc.js

@@ -143,11 +143,6 @@
     }
   },
   "actions": {
-    "raw": {
-      "title": "Raw",
-      "main": "./lib/actions/raw.js",
-      "description": "Raw"
-    },
     "queryAction": {
       "title": "Query",
       "main": "./lib/actions/query.js",

Gruntfile.js

@@ -1,40 +1,27 @@
 const { messages } = require('elasticio-node');
 const { Readable } = require('stream');
 const util = require('../util');
-
-
-const MetaLoader = require('../helpers/metaLoader');
-const sfConnection = require('../helpers/sfConnection.js');
+const { callJSForceMethod } = require('../helpers/wrapper');
 
 const DEFAULT_TIMEOUT = 600; // 10 min
 
-
 exports.objectTypes = async function objectTypes(configuration) {
-  const metaLoader = new MetaLoader(configuration, this);
-  switch (configuration.sobject) {
+  switch (configuration.operation) {
     case 'insert': {
-      return metaLoader.getCreateableObjectTypes();
+      return callJSForceMethod.call(this, configuration, 'getCreateableObjectTypes');
     }
     case 'update': {
-      return metaLoader.getUpdateableObjectTypes();
-    }
-    case 'upsert': {
-      return metaLoader.getObjectTypes();
+      return callJSForceMethod.call(this, configuration, 'getUpdateableObjectTypes');
     }
     default: {
-      // 'delete' operation or anything else
-      return metaLoader.getObjectTypes();
+      // 'delete' and 'upsert' operation or anything else
+      return callJSForceMethod.call(this, configuration, 'getObjectTypes');
     }
   }
 };
 
-
 exports.process = async function bulkCUD(message, configuration) {
-  this.logger.debug('Starting:', configuration.operation);
-
-  const conn = sfConnection.createConnection(configuration, this);
-
-  // Bulk operation ('insert', 'update', 'delete', 'upsert')
+  this.logger.info('Starting Bulk %s action', configuration.operation);
   // Get CSV from attachment
   if (!message.attachments || Object.keys(message.attachments).length === 0) {
     this.logger.error('Attachment not found');
@@ -59,15 +46,19 @@ exports.process = async function bulkCUD(message, configuration) {
   if (configuration.operation === 'upsert') {
     extra = { extIdField: message.body.extIdField };
   }
-  // Create job
-  const job = conn.bulk.createJob(configuration.sobject, configuration.operation, extra);
-  const batch = job.createBatch();
 
+  const batchOptions = {
+    sobject: configuration.sobject,
+    operation: configuration.operation,
+    extra,
+  };
+  const job = await callJSForceMethod.call(this, configuration, 'bulkCreateJob', batchOptions);
+  const batch = job.createBatch();
   return new Promise((resolve, reject) => {
     // Upload CSV to SF
     batch.execute(csvStream)
       // eslint-disable-next-line no-unused-vars
-      .on('queue', (batchInfo) => {
+      .on('queue', () => {
         // Check while job status become JobComplete or Failed, Aborted
         batch.poll(1000, timeout * 1000);
       }).on('response', (rets) => {
@@ -85,7 +76,6 @@ exports.process = async function bulkCUD(message, configuration) {
   });
 };
 
-
 exports.getMetaModel = async function getMetaModel(configuration) {
   const meta = {
     in: {

component.json

@@ -1,18 +1,11 @@
 const { messages } = require('elasticio-node');
 const client = require('elasticio-rest-node')();
-
 const request = require('request');
-
-const sfConnection = require('../helpers/sfConnection.js');
-
+const { callJSForceMethod } = require('../helpers/wrapper');
 
 exports.process = async function bulkQuery(message, configuration) {
-  this.logger.error('Starting: query');
-
-  const conn = sfConnection.createConnection(configuration, this);
-
+  this.logger.info('Starting Bulk Query action');
   const signedUrl = await client.resources.storage.createSignedUrl();
-
   const out = messages.newEmptyMessage();
   out.attachments = {
     'bulk_query.csv': {
@@ -21,23 +14,14 @@ exports.process = async function bulkQuery(message, configuration) {
     },
   };
   out.body = {};
-
+  const stream = await callJSForceMethod.call(this, configuration, 'bulkQuery', message.body.query);
   return new Promise((resolve, reject) => {
-    // Bulk operation ('query')
-    const stream = conn.bulk.query(message.body.query)
-      .on('error', (err) => {
-        this.logger.debug('error query:', err);
-        reject(err);
-      })
-      .stream();
-
-    // upload csv attachment
     stream.pipe(request.put(signedUrl.put_url, (err, resp, body) => {
       if (err) {
-        this.logger.debug('error upload:', err);
+        this.logger.error('Error upload query results');
         reject(err);
       } else {
-        this.logger.debug('success');
+        this.logger.info('Action successfully processed');
         out.body = { result: body };
         resolve(out);
       }

lib/actions/bulk_cud.js

@@ -13,15 +13,15 @@ exports.getMetaModel = async function getMetaModel(configuration) {
 };
 
 exports.process = async function createObject(message, configuration) {
-  this.logger.info(`Preparing to create a ${configuration.sobject} object...`);
-  this.logger.info('Starting Upsert Object Action');
+  this.logger.info('Starting Create Object Action');
+  this.logger.debug(`Preparing to create a ${configuration.sobject} object...`);
   const binaryField = await attachment.prepareBinaryData(message, configuration, this);
 
   this.logger.info('Sending request to SalesForce...');
   const response = await callJSForceMethod.call(this, configuration, 'sobjectCreate', message);
 
-  this.logger.debug('SF response: ', response);
-  this.logger.info(`${configuration.sobject} has been successfully created (ID = ${response.id}).`);
+  this.logger.debug(`${configuration.sobject} has been successfully created`);
+  this.logger.trace(`${configuration.sobject} has been successfully created (ID = ${response.id}).`);
   // eslint-disable-next-line no-param-reassign
   message.body.id = response.id;