Merge pull request #19 from mrodrig/add-field-option

Add KEYS option to allow for specific keys to be converted

Author: mrodrig

README.md

@@ -40,7 +40,6 @@ var converter = require('json-2-csv');
 ##### json2csv Example:
 
 ```javascript
-
 var converter = require('json-2-csv');
 
 var documents = [
@@ -70,7 +69,6 @@ var json2csvCallback = function (err, csv) {
 };
 
 converter.json2csv(documents, json2csvCallback);
-
 ```
 
 The above code prints out:
@@ -90,7 +88,10 @@ BMW,X5,2014,3287,M
     * `FIELD` - String - Field Delimiter. Default: `','`
     * `ARRAY` - String - Array Value Delimiter. Default: `';'`
   * `EOL` - String - End of Line Delimiter. Default: `'\n'`
-  * `PARSE_CSV_NUMBERS` - Boolean - Should numbers that are found in the CSV be converted to numbers? Default: `false`
+  * `PARSE_CSV_NUMBERS` - Boolean - (TODO) Should numbers that are found in the CSV be converted to numbers? Default: `false`
+  * `KEYS` - Array - Specify the keys (as strings) that should be converted. Default: `null`
+    * If you have a nested object (ie. {info : {name: 'Mike'}}), then set options.KEYS to ['info.name']
+    * If you want all keys to be converted, then specify ```null``` or don't specify the option to utilize the default.
 
 ##### csv2json Example:
 
@@ -137,19 +138,72 @@ _Note_: This requires `mocha`, `should`, `async`, and `underscore`.
 ## Features
 
 - Header Generation (per document keys)
+- Allows for conversion of specific keys in both json2csv and csv2json via the options.KEYS parameter (as of 1.1.2)
 - Verifies all documents have same schema (schema field order does not matter as of 1.1.0)
 - Supports sub-documents natively
 - Supports arrays as document values for both json2csv and csv2json
 - Custom ordering of columns (see F.A.Q. for more information)
 - Ability to re-generate the JSON documents that were used to generate the CSV (including nested documents)
 - Allows for custom field delimiters, end of line delimiters, etc.
-- Promisifiable via bluebird's .promisify(<function) and .promisifyAll() (as of 1.1.1)
+- Promisifiable via bluebird's .promisify(<function>) and .promisifyAll(<object>) (as of 1.1.1)
 
 ## F.A.Q.
 
 - Can the order of the keys be changed in the output?
 __Yes.__ Currently, changing the order of the keys in the JSON document will also change the order of the columns. (Tested on Node 10.xx)
 
+- Can I specify the keys that I would like to have converted to CSV or JSON?
+__Yes.__ This is currently supported for both json2csv and csv2json.  Specify the keys in options.KEYS. For example,
+
+```javascript
+var converter = require('json-2-csv');
+
+var options = {
+    KEYS : ['info.name', 'year']
+};
+
+var documents = [
+    {
+        "info": {
+            "name": "Mike"
+        },
+        "coursesTaken": ["CS2500", "CS2510"],
+        "year": "Sophomore"
+    },
+    {
+        "info": {
+            "name": "John"
+        },
+        "coursesTaken": ["ANTH1101", "POL2312", "MATH2142", "POL3305", "LAW2100"],
+        "year": "Senior"
+    },
+    {
+        "info": {
+                    "name": "Joe"
+        },
+        "coursesTaken": [],
+        "year": "Freshman"
+    }
+];
+
+converter.json2csv(documents, function (err, csv) {
+    if (!err) {
+        return console.log(csv);
+    }
+    throw err;
+}, options);
+```
+
+This prints out:
+
+```csv
+info.name,year
+Mike,Sophomore
+John,Senior
+Joe,Freshman
+
+```
+
 ## Milestones
  - Created: Apr 23, 2014
  - 1K Downloads/Month: January 15, 2015

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "json-2-csv",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "homepage": "https://github.com/mrodrig/json-2-csv",
   "moduleType": [
     "node"

lib/converter.js

@@ -14,7 +14,8 @@ var defaultOptions = {
         WRAP   :  ''
     },
     EOL               : '\n',
-    PARSE_CSV_NUMBERS : false
+    PARSE_CSV_NUMBERS : false,
+    KEYS              : null
 };
 
 /**

lib/csv-2-json.js

@@ -9,8 +9,14 @@ var retrieveHeading = function (lines, callback) {
     if (!lines.length) { // If there are no lines passed in, then throw an error
         return callback(new Error("No data provided to retrieve heading.")); // Pass an error back to the user
     }
-    var heading = lines[0]; // Grab the top line (header line)
-    return heading.split(options.DELIMITER.FIELD); // Return the heading split by the field delimiter
+    var heading = lines[0].split(options.DELIMITER.FIELD); // Grab the top line (header line) and split by the field delimiter
+    heading = _.map(heading, function (headerKey, index) {
+        return {
+            value: headerKey,
+            index: index
+        }
+    });
+    return heading;
 };
 
 // Add a nested key and its value in the given document
@@ -53,21 +59,18 @@ var convertArrayRepresentation = function (val) {
 };
 
 // Create a JSON document with the given keys (designated by the CSV header) and the values (from the given line)
-var createDoc = function (keys, line, callback) {
+var createDoc = function (keys, line) {
+    if (line == '') { return false; } // If we have an empty line, then return false so we can remove all blank lines (falsy values)
     var doc = {}, // JSON document to start with and manipulate
         val,      // Temporary variable to set the current key's value to
         line = line.trim().split(options.DELIMITER.FIELD); // Split the line using the given field delimiter after trimming whitespace
-    if (line == '') { return false; } // If we have an empty line, then return false so we can remove all blank lines (falsy values)
-    if (keys.length !== line.length) { // If the number of keys is different than the number of values in the current line
-        return callback(new Error("Not every line has a correct number of values.")); // Pass the error back to the client
-    }
     _.each(keys, function (key, indx) {
-        val = line[indx] === '' ? null : line[indx];
+        val = line[key.index] === '' ? null : line[key.index];
         if (isArrayRepresentation(val)) {
             val = convertArrayRepresentation(val);
         }
-        if (key.indexOf('.')) { // If key has '.' representing nested document
-            doc = addNestedKey(key, val, doc); // Update the document to add the nested key structure
+        if (key.value.indexOf('.')) { // If key has '.' representing nested document
+            doc = addNestedKey(key.value, val, doc); // Update the document to add the nested key structure
         } else { // Else we just have a straight key:value mapping
             doc[key] = val; // Set the value at the current key
         }
@@ -77,13 +80,16 @@ var createDoc = function (keys, line, callback) {
 
 // Main wrapper function to convert the CSV to the JSON document array
 var convertCSV = function (lines, callback) {
-    var headers = retrieveHeading(lines, callback), // Retrieve the headings from the CSV
-        jsonDocs = []; // Create an array that we can add the generated documents to
+    var generatedHeaders = retrieveHeading(lines, callback), // Retrieve the headings from the CSV, unless the user specified the keys
+        jsonDocs = [], // Create an array that we can add the generated documents to
+        headers = options.KEYS ? _.filter(generatedHeaders, function (headerKey) {
+            return _.contains(options.KEYS, headerKey.value);
+        }) : generatedHeaders;
     lines = lines.splice(1); // Grab all lines except for the header
     _.each(lines, function (line) { // For each line, create the document and add it to the array of documents
         jsonDocs.push(createDoc(headers, line));
     });
-    return _.filter(jsonDocs, function (doc) { return doc !== false; });; // Return all non 'falsey' values to filter blank lines
+    return _.filter(jsonDocs, function (doc) { return doc !== false; }); // Return all non 'falsey' values to filter blank lines
 };
 
 module.exports = {
@@ -103,4 +109,4 @@ module.exports = {
         callback(null, json); // Send the data back to the caller
     }
 
-};
+};

lib/json-2-csv.js

@@ -8,6 +8,7 @@ var options = {}; // Initialize the options - this will be populated when the js
 // Retrieve the headings for all documents and return it.  This checks that all documents have the same schema.
 var generateHeading = function(data) {
     return new Promise(function (resolve, reject) {
+        if (options.KEYS) { resolve(options.KEYS); }
         var keys = _.map(_.keys(data), function (key, indx) { // for each key
             if (_.isObject(data[key])) {
                 // if the data at the key is a document, then we retrieve the subHeading starting with an empty string heading and the doc

package.json

@@ -2,7 +2,7 @@
     "author": "mrodrig",
     "name": "json-2-csv",
     "description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
-    "version": "1.1.1",
+    "version": "1.1.2",
     "repository": {
         "type": "git",
     	"url": "http://github.com/mrodrig/json-2-csv.git"

test/CSV/arrayValueDocs_specificKeys.csv

@@ -0,0 +1,4 @@
+info.name,year
+Mike,Sophomore
+John,Senior
+Joe,Freshman

test/CSV/withQuotes/arrayValueDocs_specificKeys.csv

@@ -0,0 +1,4 @@
+"info.name","year"
+"Mike","Sophomore"
+"John","Senior"
+"Joe","Freshman"

test/JSON/arrayValueDocs_specificKeys.json

@@ -0,0 +1,20 @@
+[
+        {
+                "info": {
+                    "name": "Mike"
+                },
+                "year": "Sophomore"
+        },
+        {
+                "info": {
+                    "name": "John"
+                },
+                "year": "Senior"
+        },
+        {
+                "info": {
+                            "name": "Joe"
+                },
+                "year": "Freshman"
+        }
+]

test/testComma.js

@@ -21,6 +21,7 @@ var json_regularJson                 = require('./JSON/regularJson'),
     json_noData                      = require('./JSON/noData.json'),
     json_singleDoc                   = require('./JSON/singleDoc.json'),
     json_arrayValue                  = require('./JSON/arrayValueDocs.json'),
+    json_arrayValue_specificKeys     = require('./JSON/arrayValueDocs_specificKeys.json'),
     json_sameSchemaDifferentOrdering = require('./JSON/sameSchemaDifferentOrdering'),
     json_differentSchemas            = require('./JSON/differentSchemas'),
     csv_regularJson                  = '',
@@ -29,7 +30,8 @@ var json_regularJson                 = require('./JSON/regularJson'),
     csv_nestedQuotes                 = '',
     csv_noData                       = '',
     csv_singleDoc                    = '',
-    csv_arrayValue                   = '';
+    csv_arrayValue                   = '',
+    csv_arrayValue_specificKeys      = '';
 
 var json2csvTests = function () {
     describe('json2csv - non-promisified', function (done) {
@@ -90,6 +92,16 @@ var json2csvTests = function () {
                 }, options);
             });
 
+            it('should parse the specified keys to CSV', function (done) {
+                // Create a copy so we don't modify the actual options object
+                var opts = _.extend(JSON.parse(JSON.stringify(options)), {KEYS: ['info.name', 'year']});
+                converter.json2csv(json_arrayValue, function (err, csv) {
+                    csv.should.equal(csv_arrayValue_specificKeys);
+                    csv.split(options.EOL).length.should.equal(5);
+                    done();
+                }, opts);
+            });
+
             it('should parse an array of JSON documents with the same schema but different ordering of fields', function (done) {
                 converter.json2csv(json_sameSchemaDifferentOrdering, function (err, csv) {
                     csv.should.equal(csv_regularJson);
@@ -510,6 +522,20 @@ var json2csvTests = function () {
                     });
             });
 
+            it('should parse the specified keys to CSV', function (done) {
+                // Create a copy so we don't modify the actual options object
+                var opts = _.extend(JSON.parse(JSON.stringify(options)), {KEYS: ['info.name', 'year']});
+                converter.json2csvAsync(json_arrayValue, opts)
+                    .then(function (csv) {
+                        csv.should.equal(csv_arrayValue_specificKeys);
+                        csv.split(options.EOL).length.should.equal(5);
+                        done();
+                    })
+                    .catch(function (err) {
+                        throw err;
+                    });
+            });
+
             it('should parse an array of JSON documents with the same schema but different ordering of fields', function (done) {
                 converter.json2csvAsync(json_sameSchemaDifferentOrdering)
                     .then(function (csv) {
@@ -617,6 +643,15 @@ var csv2jsonTests = function () {
                 }, options);
             });
 
+            it('should parse the specified keys to JSON', function (done) {
+                var opts = _.extend(JSON.parse(JSON.stringify(options)), {KEYS : ['info.name', 'year']});
+                converter.csv2jsonAsync(csv_arrayValue.replace(/,/g, options.DELIMITER.FIELD), function (err, json) {
+                    var isEqual = _.isEqual(json, json_arrayValue_specificKeys);
+                    true.should.equal(isEqual);
+                    done();
+                }, opts);
+            });
+
             it('should throw an error about not having been passed data - 1', function (done) {
                 converter.csv2json(null, function (err, json) {
                     err.message.should.equal('Cannot call csv2json on null.');
@@ -876,6 +911,19 @@ var csv2jsonTests = function () {
                     });
             });
 
+            it('should parse the specified keys to JSON', function (done) {
+                var opts = _.extend(JSON.parse(JSON.stringify(options)), {KEYS : ['info.name', 'year']});
+                converter.csv2jsonAsync(csv_arrayValue.replace(/,/g, options.DELIMITER.FIELD), opts)
+                    .then(function (json) {
+                        var isEqual = _.isEqual(json, json_arrayValue_specificKeys);
+                        true.should.equal(isEqual);
+                        done();
+                    })
+                    .catch(function (err) {
+                        throw err;
+                    });
+            });
+
             it('should throw an error about not having been passed data - 1', function (done) {
                 converter.csv2jsonAsync(null, options)
                     .then(function (json) {
@@ -1063,6 +1111,13 @@ module.exports = {
                                 csv_arrayValue = data.toString();
                                 callback(null);
                             });
+                        },
+                        function(callback) {
+                            fs.readFile('test/CSV/arrayValueDocs_specificKeys.csv', function (err, data) {
+                                if (err) callback(err);
+                                csv_arrayValue_specificKeys = data.toString();
+                                callback(null);
+                            });
                         }
                     ],
                     function(err, results) {