Adding support for array values in docs. Updated README

mrodrig · mrodrig · commit eba1491d208b · 2014-05-04T21:43:03.000-04:00
diff --git a/README.md b/README.md
@@ -27,10 +27,12 @@ var converter = require('json-2-csv');
 
 #### json2csv(array, callback, options)
 
-* `array` - An array of JSON documents
-* `callback` - A function of the form `function (err, csv)`; This function will receive any errors and/or the CSV generated.
+* `csv` - A string of CSV
+* `callback` - A function of the form `function (err, array)`; This function will receive any errors and/or the array of JSON documents generated.
 * `options` - (Optional) A JSON document specifying any of {`DELIMITER`, `EOL`, `PARSE_CSV_NUMBERS`}
-  * `DELIMITER` - String - Field Delimiter. Default: `','`
+  * `DELIMITER` - Document - Specifies the different types of delimiters
+    * `FIELD` - String - Field Delimiter. Default: `','`
+    * `ARRAY` - String - Array Value Delimiter. Default: `';'`
   * `EOL` - String - End of Line Delimiter. Default: `'\n'`
   * `PARSE_CSV_NUMBERS` - Boolean - Should numbers that are found in the CSV be converted to numbers? Default: `false`
 
@@ -42,21 +44,21 @@ var converter = require('json-2-csv');
 
 var documents = [
     {
-        'Make': 'Nissan',
-        'Model': 'Murano',
-        'Year': '2013'
-        'Specifications': {
-            'Mileage': '7106',
-            'Trim': 'S AWD'
+        Make: 'Nissan',
+        Model: 'Murano',
+        Year: '2013'
+        Specifications: {
+            Mileage: '7106',
+            Trim: 'S AWD'
         }
     },
     {
-        'Make': 'BMW',
-        'Model' 'X5',
-        'Year': '2014',
-        'Specifications': {
-            'Mileage': '3287',
-            'Trim': 'M'
+        Make: 'BMW',
+        Model' 'X5',
+        Year: '2014',
+        Specifications: {
+            Mileage: '3287',
+            Trim: 'M'
         }
     }
 ];
@@ -83,7 +85,9 @@ BMW,X5,2014,3287,M
 * `csv` - A string of CSV
 * `callback` - A function of the form `function (err, array)`; This function will receive any errors and/or the array of JSON documents generated.
 * `options` - (Optional) A JSON document specifying any of {`DELIMITER`, `EOL`, `PARSE_CSV_NUMBERS`}
-  * `DELIMITER` - String - Field Delimiter. Default: `','`
+  * `DELIMITER` - Document - Specifies the different types of delimiters
+    * `FIELD` - String - Field Delimiter. Default: `','`
+    * `ARRAY` - String - Array Value Delimiter. Default: `';'`
   * `EOL` - String - End of Line Delimiter. Default: `'\n'`
   * `PARSE_CSV_NUMBERS` - Boolean - Should numbers that are found in the CSV be converted to numbers? Default: `false`
 
@@ -134,15 +138,17 @@ _Note_: This requires `mocha`, `should`, `async`, and `underscore`.
 - Header Generation (per document keys)
 - Verifies all documents have same schema
 - Supports sub-documents natively
+- Supports arrays as document values for both json2csv and csv2json
 - Custom ordering of columns (see F.A.Q. for more information)
 - Ability to re-generate the JSON documents that were used to generate the CSV (including nested documents)
 - Allows for custom field delimiters, end of line delimiters, etc.
 
 ## F.A.Q.
 
 - Can the order of the keys be changed in the output?
-__Yes.__ Currently, changing the order of the keys in the JSON document will also change the order of the columns. (Node 10.26)
+__Yes.__ Currently, changing the order of the keys in the JSON document will also change the order of the columns. (Tested on Node 10.xx)
 
 ## TODO
 - Use PARSE_CSV_NUMBERS option to actually convert numbers. Not currently implemented.
-- Add test cases (& fix potential issues) where data is an array of values
+- Respect nested arrays when in json2csv - Currently flattens them
+- If quotes in CSV header, strip them? Add as an option?
diff --git a/lib/converter.js b/lib/converter.js
@@ -5,26 +5,25 @@ var json2Csv = require('./json-2-csv'), // Require our json-2-csv code
     _ = require('underscore'); // Require underscore
 
 // Default options; By using a function this is essentially a 'static' variable
-var OPTIONS = function () {
-    return {
-        DELIMITER         : ',',
-        EOL               : '\n',
-        PARSE_CSV_NUMBERS : false
-    };
-}
+var defaultOptions = {
+    DELIMITER         : {
+        FIELD  :  ',',
+        ARRAY  :  ';'
+    },
+    EOL               : '\n',
+    PARSE_CSV_NUMBERS : false
+};
 
 // Build the options to be passed to the appropriate function
 // If a user does not provide custom options, then we use our default
 // If options are provided, then we set each valid key that was passed
-var buildOptions = function (opts) {
-    var out = _.extend(OPTIONS(), {});
-    if (!opts) { return out; } // If undefined or null, return defaults
-    _.each(_.keys(opts), function (key) {
-        if (out[key]) { // If key is valid, set it
-            out[key] = opts[key];
-        } // Else ignore its value
-    });
-    return out; // Return customized version
+var buildOptions = function (opts, cb) {
+    opts = opts ? opts : {}; // If undefined, set to an empty doc
+    var out = _.defaults(opts, defaultOptions);
+    // If the delimiter fields are the same, report an error to the caller
+    if (out.DELIMITER.FIELD === out.DELIMITER.ARRAY) { return cb(new Error('The field and array delimiters must differ.')); }
+    // Otherwise, send the options back
+    else { return cb(null, out); }
 };
 
 // Export the following functions that will be client accessible
@@ -35,8 +34,13 @@ module.exports = {
     // a callback that will be called with (err, csv) after
     // processing is completed, and optional options
     json2csv: function (array, callback, opts) {
-        opts = buildOptions(opts); // Build the options
-        json2Csv.json2csv(opts, array, callback); // Call our internal json2csv function
+        buildOptions(opts, function (err, options) { // Build the options
+            if (err) {
+                return callback(err);
+            } else {
+                json2Csv.json2csv(options, array, callback); // Call our internal json2csv function
+            }
+        });
     },
 
     
@@ -45,8 +49,12 @@ module.exports = {
     // a callback that will be called with (err, csv) after
     // processing is completed, and optional options
     csv2json: function (csv, callback, opts) {
-        opts = buildOptions(opts);
-        csv2Json.csv2json(opts, csv, callback);
+        buildOptions(opts, function (err, options) { // Build the options
+            if (err) {
+                return callback(err);
+            } else {
+                csv2Json.csv2json(options, csv, callback); // Call our internal csv2json function
+            }
+        });
     }
-
 };
diff --git a/lib/csv-2-json.js b/lib/csv-2-json.js
@@ -11,7 +11,7 @@ var retrieveHeading = function (lines, callback) {
         return callback(new Error("No data provided to retrieve heading.")); // Pass an error back to the user
     }
     var heading = lines[0]; // Grab the top line (header line)
-    return heading.split(options.DELIMITER); // Return the heading split by the field options.DELIMITER
+    return heading.split(options.DELIMITER.FIELD); // Return the heading split by the field delimiter
 };
 
 // Add a nested key and its value in the given document
@@ -35,19 +35,38 @@ var addNestedKey = function (key, value, doc) {
 // Helper function to check if the given key already exists in the given document
 var keyExists = function (key, doc) {
     return (typeof doc[key] !== 'undefined'); // If the key doesn't exist, then the type is 'undefined'
-}
+};
+
+var isArrayRepresentation = function (value) {
+    return (value && value.indexOf('[') === 0 && value.lastIndexOf(']') === value.length-1);
+};
+
+var convertArrayRepresentation = function (val) {
+    val = _.filter(val.substring(1, val.length-1).split(options.DELIMITER.ARRAY), function (value) {
+        return value;
+    });
+    _.each(val, function (value, indx) {
+        if (isArrayRepresentation(value)) {
+            val[indx] = convertArrayRepresentation(value);
+        }
+    });
+    return val;
+};
 
 // Create a JSON document with the given keys (designated by the CSV header) and the values (from the given line)
 var createDoc = function (keys, line, callback) {
     var doc = {}, // JSON document to start with and manipulate
         val,      // Temporary variable to set the current key's value to
-        line = line.trim().split(options.DELIMITER); // Split the line using the given DELIMITER after trimming whitespace
+        line = line.trim().split(options.DELIMITER.FIELD); // Split the line using the given field delimiter after trimming whitespace
     if (line == '') { return false; } // If we have an empty line, then return false so we can remove all blank lines (falsy values)
     if (keys.length !== line.length) { // If the number of keys is different than the number of values in the current line
         return callback(new Error("Not every line has a correct number of values.")); // Pass the error back to the client
     }
     _.each(keys, function (key, indx) {
         val = line[indx] === '' ? null : line[indx];
+        if (isArrayRepresentation(val)) {
+            val = convertArrayRepresentation(val);
+        }
         if (key.indexOf('.')) { // If key has '.' representing nested document
             doc = addNestedKey(key, val, doc); // Update the document to add the nested key structure
         } else { // Else we just have a straight key:value mapping
diff --git a/lib/json-2-csv.js b/lib/json-2-csv.js
@@ -11,22 +11,22 @@ var retrieveSubHeading = function (heading, data) {
         newKey; // temporary variable to aid in determining the heading - used to generate the 'nested' headings
     _.each(subKeys, function (subKey, indx) {
         // If the given heading is empty, then we set the heading to be the subKey, otherwise set it as a nested heading w/ a dot
-        newKey = heading === '' ? subKey : heading + '.' + subKey; 
-        if (typeof data[subKey] === 'object' && data[subKey] !== null) { // If we have another nested document
+        newKey = heading === '' ? subKey : heading + '.' + subKey;
+        if (typeof data[subKey] === 'object' && data[subKey] !== null && typeof data[subKey].length === 'undefined') { // If we have another nested document
             subKeys[indx] = retrieveSubHeading(newKey, data[subKey]); // Recur on the subdocument to retrieve the full key name
         } else {
             subKeys[indx] = newKey; // Set the key name since we don't have a sub document
         }
     });
-    return subKeys.join(options.DELIMITER); // Return the headings joined by our delimiter
+    return subKeys.join(options.DELIMITER.FIELD); // Return the headings joined by our field delimiter
 };
 
 // Retrieve the headings for all documents and return it.  This checks that all documents have the same schema.
 var retrieveHeading = function (data) {
     return function (cb) { // Returns a function that takes a callback - the function is passed to async.parallel
         var keys = _.keys(data); // Retrieve the current data keys
         _.each(keys, function (key, indx) { // for each key
-            if (typeof data[key] === 'object') { 
+            if (typeof data[key] === 'object') {
                 // if the data at the key is a document, then we retrieve the subHeading starting with an empty string heading and the doc
                 keys[indx] = retrieveSubHeading('', data[key]);
             }
@@ -35,7 +35,7 @@ var retrieveHeading = function (data) {
         keys = _.uniq(keys);
         // If we have more than 1 unique list, then not all docs have the same schema - report an error
         if (keys.length > 1) { throw new Error('Not all documents have the same schema.', keys); }
-        cb(null, _.flatten(keys).join(options.DELIMITER)); // Return headings back
+        return cb(null, _.flatten(keys).join(options.DELIMITER.FIELD)); // Return headings back
     };
 };
 
@@ -46,14 +46,16 @@ var convertData = function (data, keys) {
     _.each(keys, function (key, indx) { // For each key
         value = data[key]; // Set the current data that we are looking at
         if (keys.indexOf(key) > -1) { // If the keys contain the current key, then process the data
-            if (typeof value === 'object') { // If we have an object
+            if (typeof value === 'object' && value !== null && typeof value.length === 'undefined') { // If we have an object
                 output.push(convertData(value, _.keys(value))); // Push the recursively generated CSV
+            } else if (typeof value === 'object' && value !== null && typeof value.length === 'number') { // We have an array of values
+                output.push('[' + value.join(options.DELIMITER.ARRAY) + ']');
             } else {
                 output.push(value); // Otherwise push the current value
             }
         }
     });
-    return output.join(options.DELIMITER); // Return the data joined by our field delimiter
+    return output.join(options.DELIMITER.FIELD); // Return the data joined by our field delimiter
 };
 
 // Generate the CSV representing the given data.
@@ -74,17 +76,17 @@ module.exports = {
         else { options = opts; } // Options were passed, set the global options value
         if (!data) { callback(new Error('Cannot call json2csv on ' + data + '.')); return null; } // If we don't receive data, report an error
         if (typeof data !== 'object') { // If the data was not a single document or an array of documents
-            cb(new Error('Data provided was not an array of documents.'));  // Report the error back to the caller
+            return cb(new Error('Data provided was not an array of documents.'));  // Report the error back to the caller
         } else if (typeof data === 'object' && !data.length) { // Single document, not an array
             data = [data]; // Convert to an array of the given document
         }
         // Retrieve the heading and the CSV asynchronously in parallel
         async.parallel([retrieveHeading(data), generateCsv(data)], function (err, res) {
             if (!err) {
                 // Data received with no errors, join the two responses with an end of line delimiter to setup heading and CSV body
-                callback(null, res.join(options.EOL)); 
+                return callback(null, res.join(options.EOL));
             } else {
-                callback(err, null); // Report received error back to caller
+                return callback(err, null); // Report received error back to caller
             }
         });
     }
diff --git a/test/CSV/arrayValueDocs.csv b/test/CSV/arrayValueDocs.csv
@@ -0,0 +1,4 @@
+info.name,coursesTaken,year
+Mike,[CS2500/CS2510],Sophomore
+John,[ANTH1101/POL2312/MATH2142/POL3305/LAW2100],Senior
+Joe,[],Freshman
diff --git a/test/JSON/arrayValueDocs.json b/test/JSON/arrayValueDocs.json
@@ -0,0 +1,23 @@
+[
+        {
+                "info": {
+                    "name": "Mike"
+                },
+                "coursesTaken": ["CS2500", "CS2510"],
+                "year": "Sophomore"
+        },
+        {
+                "info": {
+                    "name": "John"
+                },
+                "coursesTaken": ["ANTH1101", "POL2312", "MATH2142", "POL3305", "LAW2100"],
+                "year": "Senior"
+        },
+        {
+                "info": {
+                            "name": "Joe"
+                },
+                "coursesTaken": [],
+                "year": "Freshman"
+        }
+]
diff --git a/test/testComma.js b/test/testComma.js
diff --git a/test/testSemi.js b/test/testSemi.js
