Fixing issue #12, added tests, bumped version up to 1.1.0

Author: mrodrig

README.md

@@ -137,7 +137,7 @@ _Note_: This requires `mocha`, `should`, `async`, and `underscore`.
 ## Features
 
 - Header Generation (per document keys)
-- Verifies all documents have same schema
+- Verifies all documents have same schema (schema field order does not matter as of 1.1.0)
 - Supports sub-documents natively
 - Supports arrays as document values for both json2csv and csv2json
 - Custom ordering of columns (see F.A.Q. for more information)
@@ -151,9 +151,4 @@ __Yes.__ Currently, changing the order of the keys in the JSON document will als
 
 ## Milestones
  - Created: Apr 23, 2014
- - 1K Downloads/Month: January 15, 2015
-
-## TODO
-- Use PARSE_CSV_NUMBERS option to actually convert numbers. Not currently implemented.
-- Respect nested arrays when in json2csv - Currently flattens them
-- If quotes in CSV header, strip them? Add as an option?
+ - 1K Downloads/Month: January 15, 2015

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "json-2-csv",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "homepage": "https://github.com/mrodrig/json-2-csv",
   "moduleType": [
     "node"
@@ -15,6 +15,6 @@
   ],
   "dependencies": {
     "underscore": "1.6.0",
-    "async": "0.2.10"
+    "bluebird": "2.9.24"
   }
 }

lib/csv-2-json.js

@@ -1,7 +1,6 @@
 'use strict';
 
-var _ = require('underscore'),
-    async = require('async');
+var _ = require('underscore');
 
 var options = {}; // Initialize the options - this will be populated when the csv2json function is called.
 

lib/json-2-csv.js

@@ -1,33 +1,37 @@
 'use strict';
 
 var _ = require('underscore'),
-    async = require('async');
+    Promise = require('bluebird');
 
 var options = {}; // Initialize the options - this will be populated when the json2csv function is called.
 
 // Retrieve the headings for all documents and return it.  This checks that all documents have the same schema.
-var generateHeading = function(data, cb) {
-    var keys = _.map(_.keys(data), function (key, indx) { // for each key
-        if (_.isObject(data[key])) {
-            // if the data at the key is a document, then we retrieve the subHeading starting with an empty string heading and the doc
-            return generateSubHeading('', data[key]);
-        }
-        return key;
-    });
-
-    // TODO: check for consistent schema
+var generateHeading = function(data) {
+    return new Promise(function (resolve, reject) {
+        var keys = _.map(_.keys(data), function (key, indx) { // for each key
+            if (_.isObject(data[key])) {
+                // if the data at the key is a document, then we retrieve the subHeading starting with an empty string heading and the doc
+                return generateSubHeading('', data[key]);
+            }
+            return key;
+        });
 
-    keys = _.map(keys, function(keyList) {
-        return _.flatten(keyList).join(options.DELIMITER.FIELD);
+        // Check for a consistent schema that does not require the same order:
+        // if we only have one document - then there is no possiblility of multiple schemas
+        if (keys && keys.length <= 1) {
+            return resolve(_.flatten(keys) || []);
+        }
+        // else - multiple documents - ensure only one schema (regardless of field ordering)
+        var firstDocSchema = _.flatten(keys[0]);
+        _.each(keys, function (keyList) {
+            // If there is a difference between the schemas, throw the inconsistent schema error
+            var diff = _.difference(firstDocSchema, _.flatten(keyList));
+            if (!_.isEqual(diff, [])) {
+                return reject(new Error('Not all documents have the same schema.'));
+            }
+        });
+        return resolve(_.flatten(keys[0]));
     });
-
-    // Retrieve the unique array of headings (keys)
-    keys = _.uniq(keys);
-
-    // If we have more than 1 unique list, then not all docs have the same schema - report an error
-    if (keys.length > 1) { throw new Error('Not all documents have the same schema.', keys); }
-
-    return cb(null, keys);
 };
 
 // Takes the parent heading and this doc's data and creates the subdocument headings (string)
@@ -41,7 +45,7 @@ var generateSubHeading = function(heading, data) {
         if (_.isObject(data[subKey]) && !_.isNull(data[subKey]) && _.isUndefined(data[subKey].length) && _.keys(data[subKey]).length > 0) { // If we have another nested document
             return generateSubHeading(newKey, data[subKey]); // Recur on the sub-document to retrieve the full key name
         } else {
-            return options.DELIMITER.WRAP + newKey + options.DELIMITER.WRAP; // Set the key name since we don't have a sub document
+            return newKey; // Set the key name since we don't have a sub document
         }
     });
 
@@ -52,28 +56,40 @@ var generateSubHeading = function(heading, data) {
 var convertData = function (data, keys) {
     var output = [], // Array of CSV representing converted docs
         value; // Temporary variable to store the current data
-    _.each(keys, function (key, indx) { // For each key
-        value = data[key]; // Set the current data that we are looking at
-        if (keys.indexOf(key) > -1) { // If the keys contain the current key, then process the data
-            if (_.isArray(value)) { // We have an array of values
-                output.push((options.DELIMITER.WRAP || '') + '[' + value.join(options.DELIMITER.ARRAY) + ']' + (options.DELIMITER.WRAP || ''));
-            } else if (_.isDate(value)) { // If we have a date
-                output.push(value.toString());
-            } else if (_.isObject(value)) { // If we have an object
-                output.push(convertData(value, _.keys(value))); // Push the recursively generated CSV
-            } else {
-                value = value == null ? '' : value.toString();
-                output.push((options.DELIMITER.WRAP || '') + value + (options.DELIMITER.WRAP || '')); // Otherwise push the current value
-            }
+
+    _.each(keys, function (key) { // For each key
+        var indexOfPeriod = _.indexOf(key, '.');
+        if (indexOfPeriod > -1) {
+            var pathPrefix = key.slice(0, indexOfPeriod),
+                pathRemainder = key.slice(indexOfPeriod+1);
+            output.push(convertData(data[pathPrefix], [pathRemainder]));
+        } else if (keys.indexOf(key) > -1) { // If the keys contain the current key, then process the data
+            value = data[key]; // Set the current data that we are looking at
+            convertField(value, output);
         }
     });
-    return output.join(options.DELIMITER.FIELD); // Return the data joined by our field delimiter
+    return output; // Return the data joined by our field delimiter
+};
+
+var convertField = function (value, output) {
+    if (_.isArray(value)) { // We have an array of values
+        output.push(options.DELIMITER.WRAP + '[' + value.join(options.DELIMITER.ARRAY) + ']' + options.DELIMITER.WRAP);
+    } else if (_.isDate(value)) { // If we have a date
+        output.push(value.toString());
+    } else if (_.isObject(value)) { // If we have an object
+        output.push(convertData(value, _.keys(value))); // Push the recursively generated CSV
+    } else {
+        value = value === null ? '' : value.toString();
+        output.push(options.DELIMITER.WRAP + value + options.DELIMITER.WRAP); // Otherwise push the current value
+    }
 };
 
 // Generate the CSV representing the given data.
-var generateCsv = function (data, cb) {
+var generateCsv = function (data, headingKeys) {
     // Reduce each JSON document in data to a CSV string and append it to the CSV accumulator
-    return cb(null, _.reduce(data, function (csv, doc) { return csv += convertData(doc, _.keys(doc)) + options.EOL; }, ''));
+    return Promise.resolve([headingKeys, _.reduce(data, function (csv, doc) {
+        return csv += _.flatten(convertData(doc, headingKeys)).join(options.DELIMITER.FIELD) + options.EOL;
+    }, '')]);
 };
 
 module.exports = {
@@ -82,24 +98,33 @@ module.exports = {
     // Takes options as a document, data as a JSON document array, and a callback that will be used to report the results
     json2csv: function (opts, data, callback) {
         if (!callback) { throw new Error('A callback is required!'); } // If a callback wasn't provided, throw an error
+
         if (!opts) { return callback(new Error('Options were not passed and are required.')); } // Shouldn't happen, but just in case
         else { options = opts; } // Options were passed, set the global options value
+
         if (!data) { return callback(new Error('Cannot call json2csv on ' + data + '.')); } // If we don't receive data, report an error
+
         if (!_.isObject(data)) { // If the data was not a single document or an array of documents
             return callback(new Error('Data provided was not an array of documents.'));  // Report the error back to the caller
         } else if (_.isObject(data) && !data.length) { // Single document, not an array
             data = [data]; // Convert to an array of the given document
         }
 
-        // Retrieve the heading and the CSV asynchronously in parallel
-        async.parallel([_.partial(generateHeading, data), _.partial(generateCsv, data)], function (err, res) {
-            if (!err) {
-                // Data received with no errors, join the two responses with an end of line delimiter to setup heading and CSV body
-                return callback(null, res.join(options.EOL));
-            } else {
-                return callback(err, null); // Report received error back to caller
-            }
-        });
+        // Retrieve the heading and then generate the CSV with the keys that are identified
+        generateHeading(data)
+            .then(_.partial(generateCsv, data))
+            .spread(function (csvHeading, csvData) {
+                if (options.DELIMITER.WRAP) {
+                    csvHeading = _.map(csvHeading, function(headingKey) {
+                        return options.DELIMITER.WRAP + headingKey + options.DELIMITER.WRAP;
+                    });
+                }
+                csvHeading = csvHeading.join(options.DELIMITER.FIELD);
+                return callback(null, [csvHeading, csvData].join(options.EOL));
+            })
+            .catch(function (err) {
+                return callback(err);
+            });
     }
 
 };

package.json

@@ -2,7 +2,7 @@
     "author": "mrodrig",
     "name": "json-2-csv",
     "description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
-    "version": "1.0.8",
+    "version": "1.1.0",
     "repository": {
         "type": "git",
     	"url": "http://github.com/mrodrig/json-2-csv.git"
@@ -24,7 +24,7 @@
     ],
     "dependencies": {
     	"underscore": "1.6.0",
-    	"async": "0.2.10"
+        "bluebird": "~2.9.24"
     },
     "devDependencies": {
         "mocha": "~1.14.0",
@@ -35,4 +35,4 @@
     	"node": "*"
     },
     "license": "MIT"
-}
+}

test/JSON/differentSchemas.json

@@ -0,0 +1,6 @@
+[
+    { "carModel" : "Audi",      "price" : "10000",  "color" : "blue", "mileage" : "7200" },
+    { "carModel" : "BMW",       "price" : "15000",  "color" : "red" },
+    { "carModel" : "Mercedes",  "price" : "20000",  "color" : "yellow" },
+    { "carModel" : "Porsche",   "price" : "30000",  "color" : "green" }
+]

test/JSON/sameSchemaDifferentOrdering.json

@@ -0,0 +1,6 @@
+[
+    { "carModel" : "Audi",      "price" : "10000",  "color" : "blue" },
+    { "carModel" : "BMW",       "color" : "red",    "price" : "15000" },
+    { "price" : "20000",        "color" : "yellow", "carModel" : "Mercedes" },
+    { "carModel" : "Porsche",   "price" : "30000",  "color" : "green" }
+]